[DO NOT COMMIT][mlir] modernizer for ODS OpBuilder syntax

[ftynse]

Summary

This is a temporary tool to update the syntax of OpBuilder in ODS so that it
uses the dag-based version. It is based on a relatively simple parser and
formatter, which may require minor manual corrections.

Usage: mlir-tblgen-modernize filename.td

[ftynse]

Support more cases for $_builder and $_state

llvm/include/llvm/TableGen/Parser.h

+#ifndef LLVM_TABLEGEN_PARSER_H_
+#define LLVM_TABLEGEN_PARSER_H_
+
+namespace llvm {
+
+class Error;
+class StringRef;
+class RecordKeeper;
+
+Error TableGenParseString(StringRef String, RecordKeeper &Records);
+
+}  // namespace llvm
+
+#endif  // LLVM_TABLEGEN_PARSER_H_

llvm/lib/TableGen/TGParser.cpp

 //===- TGParser.cpp - Parser for TableGen Files ---------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // Implement the Parser for TableGen.
 //
 //===----------------------------------------------------------------------===//
 
 #include "TGParser.h"
 #include "llvm/ADT/None.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Compiler.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/SourceMgr.h"
+#include "llvm/TableGen/Parser.h"
 #include <algorithm>
 #include <cassert>
 #include <cstdint>
 
 using namespace llvm;
 
 //===----------------------------------------------------------------------===//
 // Support Code for the Semantic Actions.
LLVM_DUMP_METHOD void MultiClass::dump() const {
   errs() << "Record:\n";
   Rec.dump();
 
   errs() << "Defs:\n";
   for (const auto &E : Entries)
     E.dump();
 }
 #endif
+
+Error llvm::TableGenParseString(StringRef String, RecordKeeper &Records) {
+  SourceMgr SrcMgr;
+  SrcMgr.AddNewSourceBuffer(MemoryBuffer::getMemBufferCopy(String),
+                            llvm::SMLoc());
+  TGParser Parser(SrcMgr, None, Records);
+  if (Parser.ParseFile())
+    return createStringError(inconvertibleErrorCode(), "parser failure");
+  return Error::success();
+}

mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td

def LLVM_ZeroResultOpBuilder : OpBuilder<
   "ValueRange operands, ArrayRef<NamedAttribute> attributes = {}",
   [{
     $_state.addOperands(operands);
     for (auto namedAttr : attributes) {
       $_state.addAttribute(namedAttr.first, namedAttr.second);
     }
   }]>;
 
-class LLVM_TwoBuilders<OpBuilder b1, OpBuilder b2> {
-  list<OpBuilder> builders = [b1, b2];
+class LLVM_TwoBuilders<OpBuilderBase b1, OpBuilderBase b2> {
+  list<OpBuilderBase> builders = [b1, b2];
 }
 
 // Base class for LLVM operations with one result.
 class LLVM_OneResultOp<string mnemonic, list<OpTrait> traits = []> :
     LLVM_Op<mnemonic, traits>, Results<(outs LLVM_Type:$res)> {
   let builders = [LLVM_OneResultOpBuilder];
 }
 

mlir/tools/CMakeLists.txt

 add_subdirectory(mlir-cuda-runner)
 add_subdirectory(mlir-cpu-runner)
 add_subdirectory(mlir-linalg-ods-gen)
 add_subdirectory(mlir-opt)
 add_subdirectory(mlir-reduce)
 add_subdirectory(mlir-rocm-runner)
 add_subdirectory(mlir-shlib)
 add_subdirectory(mlir-spirv-cpu-runner)
+add_subdirectory(mlir-tblgen-modernize)
 add_subdirectory(mlir-translate)
-add_subdirectory(mlir-vulkan-runner)
+add_subdirectory(mlir-vulkan-runner)
 

mlir/tools/mlir-tblgen-modernize/CMakeLists.txt

+set(LLVM_LINK_COMPONENTS
+  Support
+  TableGen
+)
+
+add_llvm_tool(mlir-tblgen-modernize
+  mlir-tblgen-modernize.cpp
+
+  DEPENDS
+  MLIRSupport
+)
+target_link_libraries(mlir-tblgen-modernize
+  PRIVATE
+  MLIRSupport
+)
+llvm_update_compile_flags(mlir-tblgen-modernize)
+

mlir/tools/mlir-tblgen-modernize/mlir-tblgen-modernize.cpp

+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Parser.h"
+#include "llvm/TableGen/Record.h"
+#include "mlir/Support/FileUtilities.h"
+
+#include <string>
+
+struct ArgumentSignature {
+  llvm::StringRef type;
+  llvm::StringRef name;
+  llvm::StringRef initializer;
+};
+
+static ArgumentSignature splitSignature(llvm::StringRef signature) {
+  signature = signature.trim();
+
+  // First, check if there is an initializer. We find it by looking for the
+  // first '=' because these are unlikely (althought not impossible) to find it
+  // in the type signature.
+  size_t assignmentPos = signature.find('=');
+  llvm::StringRef initializer;
+  if (assignmentPos != std::string::npos) {
+    initializer = signature.drop_front(assignmentPos + 1).ltrim();
+    signature = signature.take_front(assignmentPos).rtrim();
+  }
+
+  for (int64_t pos = signature.size() - 1; pos >= 0; --pos) {
+    // Iterate until we find a symbol that is clearly not a part of an
+    // identifier.
+    char c = signature[pos];
+    if (llvm::isAlnum(c) || c == '_')
+      continue;
+
+    // If we found a `:`, we are guaranteed to deal with a type.
+    if (c == ':')
+      return {signature, llvm::StringRef(), initializer};
+
+    // The part may be a name or a reserved keyword that belongs to the type,
+    // e.g., in "Type *const" the "const" part belongs to the type. Check if the
+    // identifier is a keyword or not before forming the type/name pair.
+    llvm::StringRef maybeName = signature.drop_front(pos + 1);
+    if (maybeName.trim() == "const" || maybeName.trim() == "volatile")
+      return {signature, llvm::StringRef(), initializer};
+    return {signature.take_front(pos + 1).rtrim(),
+            signature.drop_front(pos + 1), initializer};
+  }
+
+  // If we only found characters that can form identifier, it's a type
+  // identifier rather than name.
+  return {signature, llvm::StringRef(), initializer};
+}
+
+// The builder signature is a comma-separated argument list. But it can also
+// have commas inside template argument list. Filter those out.
+static void
+splitBuilderSignature(llvm::StringRef signature,
+                      llvm::SmallVectorImpl<llvm::StringRef> &arguments) {
+  // Avoid inserting an empty substring.
+  if (signature.trim().empty())
+    return;
+
+  size_t numAngleBrackets = 0;
+  size_t start = 0;
+  for (size_t pos = 0; pos < signature.size(); ++pos) {
+    if (signature[pos] == '<') {
+      ++numAngleBrackets;
+    } else if (signature[pos] == '>') {
+      assert(numAngleBrackets > 0 && "unexpected syntax error");
+      --numAngleBrackets;
+    } else if (signature[pos] == ',' && numAngleBrackets == 0) {
+      arguments.push_back(signature.substr(start, pos - start));
+      start = pos + 1;
+    }
+  }
+  arguments.push_back(signature.drop_front(start));
+}
+
+llvm::StringRef
+printWithPositionedReplacement(llvm::StringRef body, size_t pos,
+                               size_t length, llvm::StringRef value) {
+  llvm::outs() << body.take_front(pos) << value;
+  return body.drop_front(pos + length);
+}
+
+void printWithReplacements(llvm::StringRef body, llvm::StringRef pattern1,
+                           llvm::StringRef value1, llvm::StringRef pattern2,
+                           llvm::StringRef value2) {
+  size_t p1size = pattern1.size();
+  size_t p2size = pattern2.size();
+
+  auto isAlnumUnder = [](char c) { return llvm::isAlnum(c) || c == '_'; };
+
+  do {
+    // Find the pattern (name) and make sure it's not preceded or followed by a
+    // character in [A-Za-z0-9_], i.e. the pattern is not inside a larger
+    // identifier.
+    size_t pos1 = body.find(pattern1);
+    if (pos1 != std::string::npos) {
+      if (pos1 + p1size < body.size() && isAlnumUnder(body[pos1 + p1size]))
+        pos1 = std::string::npos;
+      if (pos1 != 0 && isAlnumUnder(body[pos1 - 1]))
+        pos1 = std::string::npos;
+    }
+
+    size_t pos2 = body.find(pattern2);
+    if (pos2 != std::string::npos) {
+      if (pos2 + p2size < body.size() && isAlnumUnder(body[pos2 + p2size]))
+        pos2 = std::string::npos;
+      if (pos2 != 0 && isAlnumUnder(body[pos2 - 1]))
+        pos2 = std::string::npos;
+    }
+
+    if (pos1 == std::string::npos && pos2 == std::string::npos) {
+      llvm::outs() << body;
+      return;
+    }
+    body = pos1 < pos2
+        ? printWithPositionedReplacement(body, pos1, pattern1.size(), value1)
+        : printWithPositionedReplacement(body, pos2, pattern2.size(), value2);
+  } while (true);
+}
+
+std::string transformSignature(const ArgumentSignature &signature) {
+  std::string optionalName = signature.name.trim().empty()
+                                 ? ""
+                                 : llvm::formatv(":${0}", signature.name).str();
+
+  if (!signature.initializer.empty()) {
+    std::string initializer;
+    initializer.reserve(signature.initializer.size());
+    for (char c : signature.initializer) {
+      if (c == '"')
+        initializer.push_back('\\');
+      initializer.push_back(c);
+    }
+    return llvm::formatv("CArg<\"{0}\", \"{1}\">{2}", signature.type,
+                         initializer, optionalName);
+  }
+
+  return llvm::formatv("\"{0}\"{1}", signature.type, optionalName);
+}
+
+void transformBuilder(llvm::StringRef builder, size_t indent) {
+  std::string input;
+  llvm::raw_string_ostream stream(input);
+  stream << R"TG(
+    class OpBuilder<string p, code b = [{}]> {
+      string params = p;
+      code body = b;
+    }
+    def : )TG" << builder << ";";
+  stream.flush();
+
+  llvm::RecordKeeper records;
+  if (llvm::TableGenParseString(input, records)) {
+    llvm::PrintFatalError("failed to parse builder");
+    return;
+  }
+
+  for (const auto &def : records.getDefs()) {
+    if (!def.second->isSubClassOf("OpBuilder"))
+      continue;
+
+    llvm::SmallVector<llvm::StringRef, 8> paramStrings;
+    splitBuilderSignature(def.second->getValueAsString("params"), paramStrings);
+    auto params = llvm::to_vector<8>(llvm::map_range(
+        paramStrings, [](llvm::StringRef p) { return splitSignature(p); }));
+    llvm::StringRef builderOrigName, stateOrigName;
+    if (!params.empty() && params[0].type.contains("OpBuilder")) {
+      builderOrigName = params[0].name;
+      stateOrigName = params[1].name;
+      params.erase(params.begin(), std::next(params.begin(), 2));
+    }
+
+    llvm::outs() << "\n";
+    llvm::outs().indent(indent) << "OpBuilderDAG<(ins";
+    size_t prefixWidth = indent + strlen("OpBuilderDag<(ins ");
+    for (size_t i = 0, e = params.size(); i < e; ++i) {
+      std::string signature = transformSignature(params[i]);
+      if (i == 0)
+        signature = " " + signature;
+
+      if (prefixWidth + signature.size() + (i == 0 ? 0 : 2) > 80) {
+        if (i != 0)
+          llvm::outs() << ",";
+        llvm::outs() << "\n";
+        llvm::outs().indent(indent + 2) << signature;
+        prefixWidth = indent + 2 + signature.size();
+      } else {
+        if (i != 0)
+          llvm::outs() << ", ";
+        llvm::outs() << signature;
+        prefixWidth += signature.size() + (i == 0 ? 0 : 2);
+      }
+    }
+
+    llvm::outs() << ")";
+    llvm::StringRef body = def.second->getValueAsString("body");
+    if (!body.empty()) {
+      llvm::outs() << ",\n";
+      llvm::outs().indent(indent) << "[{";
+      if (!builderOrigName.empty())
+        printWithReplacements(body, builderOrigName.str(), "$_builder",
+                              stateOrigName.str(), "$_state");
+      else
+        llvm::outs() << body;
+      llvm::outs() << "}]";
+    }
+    llvm::outs() << ">";
+  }
+}
+
+int main(int argc, char **argv) {
+  llvm::cl::opt<std::string> inputFilename(llvm::cl::Positional,
+                                           llvm::cl::desc("<input file>"),
+                                           llvm::cl::init("-"));
+  llvm::InitLLVM init(argc, argv);
+  llvm::cl::ParseCommandLineOptions(argc, argv, "MLIR tablegen modernizer");
+
+  std::string errorMessage;
+  auto inputFile = mlir::openInputFile(inputFilename, &errorMessage);
+  if (!inputFile) {
+    llvm::errs() << errorMessage;
+    return EXIT_FAILURE;
+  }
+
+  llvm::StringRef buffer = inputFile->getBuffer();
+  do {
+    size_t pos = buffer.find("OpBuilder<");
+    if (pos == std::string::npos)
+      break;
+
+    // Find the leading indentation.
+    size_t indentation = 0;
+    size_t lineStart = buffer.take_front(pos).rfind('\n');
+    lineStart = lineStart == std::string::npos ? 0 : lineStart + 1;
+    for (size_t i = lineStart; i < pos; ++i) {
+      if (buffer[i] == ' ')
+        indentation += 1;
+      else if (buffer[i] == '\t')
+        indentation += 2;
+      else
+        break;
+    }
+    if (!buffer.take_front(pos).drop_front(lineStart).trim().empty())
+      indentation += 2;
+
+    // Find the matching '>'. Maintain a counter of unclosed <> pairs. Ignore <>
+    // symbols inside quoted strings or [{}] code. Also ignore any backslash
+    // escaped symbol.
+    size_t numOpen = 1;
+    bool inQuotes = false;
+    bool inCode = false;
+    size_t prefixLength = llvm::StringRef("OpBuilder<").size();
+    size_t endPos = pos + prefixLength;
+    bool skipNextChar = false;
+    for (size_t endBuffer = buffer.size(); endPos < endBuffer; ++endPos) {
+      if (skipNextChar) {
+        skipNextChar = false;
+        continue;
+      }
+
+      if (buffer[endPos] == '<' && !inQuotes && !inCode) {
+        ++numOpen;
+      } else if (buffer[endPos] == '>' && !inQuotes && !inCode) {
+        --numOpen;
+        if (numOpen == 0)
+          break;
+      } else if (buffer[endPos] == '\\') {
+        skipNextChar = true;
+      } else if (buffer[endPos] == '"' && !inCode) {
+        inQuotes = !inQuotes;
+      } else if (buffer[endPos] == '[' && endPos != endBuffer - 1 &&
+                 buffer[endPos + 1] == '{' && !inQuotes) {
+        inCode = true;
+        ++endPos;
+      } else if (buffer[endPos] == '}' && endPos != endBuffer - 1 &&
+                 buffer[endPos + 1] == ']' && !inQuotes) {
+        inCode = false;
+        ++endPos;
+      }
+    }
+
+    llvm::StringRef verbatim = buffer.take_front(pos);
+    llvm::outs() << verbatim.rtrim();
+    transformBuilder(buffer.substr(pos, endPos - pos + 1), indentation);
+    buffer = buffer.drop_front(endPos + 1);
+  } while (true);
+  llvm::outs() << buffer;
+
+  return EXIT_SUCCESS;
+}