[mlir] Extend standalone example by standalone-translate

[marbre]

Summary

Extend the standalone by standalone-translate, based on mlir-translate.

[stephenneuendorffer]

Needs a test?

[Kayjukh]

mlir/examples/standalone/standalone-translate/CMakeLists.txt

9

This should probably be replaced by add_llvm_executable.

[marbre]

In D80737#2060574, @stephenneuendorffer wrote:

Needs a test?

There is no test for iree-translate and we don't have an additional translation registered in the standalone dialect. Anyway, I could go for a test similar to https://github.com/llvm/llvm-project/blob/master/mlir/test/mlir-opt/commandline.mlir, for example:

// RUN: standalone-translate --help | FileCheck %s
// CHECK: --avx512-mlir-to-llvmir
// CHECK: --deserialize-spirv
// CHECK: --import-llvm
// CHECK: --mlir-to-llvmir
// CHECK: --mlir-to-nvvmir
// CHECK: --mlir-to-rocdlir
// CHECK: --serialize-spirv

Would that be suitable?

[marbre]

Replace add_llvm_tool by add_llvm_executable

[stephenneuendorffer]

Would that be suitable?

Yes, something like that is probably good enough for a smoke test.

[Kayjukh]

LGTM too, after adding the test case. Thanks!

[marbre]

Add standalone-translate test, rename test from check-standalone-opt to check-standalone

mlir/examples/standalone/CMakeLists.txt

 include_directories(${PROJECT_BINARY_DIR}/include)
 link_directories(${LLVM_BUILD_LIBRARY_DIR})
 add_definitions(${LLVM_DEFINITIONS})
 
 add_subdirectory(include)
 add_subdirectory(lib)
 add_subdirectory(test)
 add_subdirectory(standalone-opt)
+add_subdirectory(standalone-translate)

mlir/examples/standalone/README.md

 # An out-of-tree MLIR dialect
 
 This is an example of an out-of-tree [MLIR](https://mlir.llvm.org/) dialect along with a standalone `opt`-like tool to operate on that dialect.
 
 ## Building
 
 This setup assumes that you have built LLVM and MLIR in `$BUILD_DIR` and installed them to `$PREFIX`. To build and launch the tests, run
 ```sh
 mkdir build && cd build
 cmake -G Ninja .. -DMLIR_DIR=$PREFIX/lib/cmake/mlir -DLLVM_EXTERNAL_LIT=$BUILD_DIR/bin/llvm-lit
-cmake --build . --target check-standalone-opt
+cmake --build . --target check-standalone
 ```
 To build the documentation from the TableGen description of the dialect operations, run
 ```sh
 cmake --build . --target mlir-doc
 ```
 **Note**: Make sure to pass `-DLLVM_INSTALL_UTILS=ON` when building LLVM with CMake in order to install `FileCheck` to the chosen installation prefix.
 

mlir/examples/standalone/standalone-translate/CMakeLists.txt

+set(LLVM_LINK_COMPONENTS
+  Support
+  )
+
+get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
+get_property(translation_libs GLOBAL PROPERTY MLIR_TRANSLATION_LIBS)
+
+add_llvm_executable(standalone-translate
+  standalone-translate.cpp
+  )
+llvm_update_compile_flags(standalone-translate)
+target_link_libraries(standalone-translate
+  PRIVATE
+  ${dialect_libs}
+  ${translation_libs}
+  MLIRIR
+  MLIRParser
+  MLIRPass
+  MLIRSPIRV
+  MLIRTranslation
+  MLIRSupport
+  )
+
+mlir_check_link_libraries(standalone-translate)

mlir/examples/standalone/standalone-translate/standalone-translate.cpp

+//===- standalone-translate.cpp ---------------------------------*- C++ -*-===//
+//
+// This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This is a command line utility that translates a file from/to MLIR using one
+// of the registered translations.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/IR/AsmState.h"
+#include "mlir/IR/Diagnostics.h"
+#include "mlir/IR/MLIRContext.h"
+#include "mlir/InitAllDialects.h"
+#include "mlir/InitAllTranslations.h"
+#include "mlir/Support/FileUtilities.h"
+#include "mlir/Support/LogicalResult.h"
+#include "mlir/Support/ToolUtilities.h"
+#include "mlir/Translation.h"
+#include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/ToolOutputFile.h"
+
+#include "Standalone/StandaloneDialect.h"
+
+static llvm::cl::opt<std::string> inputFilename(llvm::cl::Positional,
+                                                llvm::cl::desc("<input file>"),
+                                                llvm::cl::init("-"));
+
+static llvm::cl::opt<std::string>
+    outputFilename("o", llvm::cl::desc("Output filename"),
+                   llvm::cl::value_desc("filename"), llvm::cl::init("-"));
+
+static llvm::cl::opt<bool>
+    splitInputFile("split-input-file",
+                   llvm::cl::desc("Split the input file into pieces and "
+                                  "process each chunk independently"),
+                   llvm::cl::init(false));
+
+static llvm::cl::opt<bool> verifyDiagnostics(
+    "verify-diagnostics",
+    llvm::cl::desc("Check that emitted diagnostics match "
+                   "expected-* lines on the corresponding line"),
+    llvm::cl::init(false));
+
+int main(int argc, char **argv) {
+  mlir::registerAllDialects();
+  mlir::registerAllTranslations();
+
+  mlir::registerDialect<mlir::standalone::StandaloneDialect>();
+  // TODO: Register standalone translations here.
+
+  llvm::InitLLVM y(argc, argv);
+
+  // Add flags for all the registered translations.
+  llvm::cl::opt<const mlir::TranslateFunction *, false, mlir::TranslationParser>
+      translationRequested("", llvm::cl::desc("Translation to perform"),
+                           llvm::cl::Required);
+  mlir::registerAsmPrinterCLOptions();
+  mlir::registerMLIRContextCLOptions();
+  llvm::cl::ParseCommandLineOptions(argc, argv, "MLIR translation driver\n");
+
+  std::string errorMessage;
+  auto input = mlir::openInputFile(inputFilename, &errorMessage);
+  if (!input) {
+    llvm::errs() << errorMessage << "\n";
+    return 1;
+  }
+
+  auto output = mlir::openOutputFile(outputFilename, &errorMessage);
+  if (!output) {
+    llvm::errs() << errorMessage << "\n";
+    return 1;
+  }
+
+  // Processes the memory buffer with a new MLIRContext.
+  auto processBuffer = [&](std::unique_ptr<llvm::MemoryBuffer> ownedBuffer,
+                           llvm::raw_ostream &os) {
+    mlir::MLIRContext context;
+    context.allowUnregisteredDialects();
+    context.printOpOnDiagnostic(!verifyDiagnostics);
+    llvm::SourceMgr sourceMgr;
+    sourceMgr.AddNewSourceBuffer(std::move(ownedBuffer), llvm::SMLoc());
+
+    if (!verifyDiagnostics) {
+      mlir::SourceMgrDiagnosticHandler sourceMgrHandler(sourceMgr, &context);
+      return (*translationRequested)(sourceMgr, os, &context);
+    }
+
+    // In the diagnostic verification flow, we ignore whether the translation
+    // failed (in most cases, it is expected to fail). Instead, we check if the
+    // diagnostics were produced as expected.
+    mlir::SourceMgrDiagnosticVerifierHandler sourceMgrHandler(sourceMgr,
+                                                              &context);
+    (*translationRequested)(sourceMgr, os, &context);
+    return sourceMgrHandler.verify();
+  };
+
+  if (splitInputFile) {
+    if (failed(mlir::splitAndProcessBuffer(std::move(input), processBuffer,
+                                           output->os())))
+      return 1;
+  } else {
+    if (failed(processBuffer(std::move(input), output->os())))
+      return 1;
+  }
+
+  output->keep();
+  return 0;
+}

mlir/examples/standalone/test/CMakeLists.txt

 configure_lit_site_cfg(
         ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
         ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
         MAIN_CONFIG
         ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py
 )
 
-set(STANDALONE_OPT_TEST_DEPENDS
+set(STANDALONE_TEST_DEPENDS
         FileCheck count not
         standalone-opt
+        standalone-translate
         )
 
-add_lit_testsuite(check-standalone-opt "Running the standalone-opt regression tests"
+add_lit_testsuite(check-standalone "Running the standalone regression tests"
         ${CMAKE_CURRENT_BINARY_DIR}
-        DEPENDS ${STANDALONE_OPT_TEST_DEPENDS}
+        DEPENDS ${STANDALONE_TEST_DEPENDS}
         )
-set_target_properties(check-standalone-opt PROPERTIES FOLDER "Tests")
+set_target_properties(check-standalone PROPERTIES FOLDER "Tests")
 
-add_lit_testsuites(STANDALONE_OPT ${CMAKE_CURRENT_SOURCE_DIR} DEPENDS ${STANDALONE_OPT_TEST_DEPENDS})
+add_lit_testsuites(STANDALONE ${CMAKE_CURRENT_SOURCE_DIR} DEPENDS ${STANDALONE_TEST_DEPENDS})

mlir/examples/standalone/test/Standalone/standalone-translate.mlir

+// RUN: standalone-translate --help | FileCheck %s
+// CHECK: --avx512-mlir-to-llvmir
+// CHECK: --deserialize-spirv
+// CHECK: --import-llvm
+// CHECK: --mlir-to-llvmir
+// CHECK: --mlir-to-nvvmir
+// CHECK: --mlir-to-rocdlir
+// CHECK: --serialize-spirv

mlir/examples/standalone/test/lit.cfg.py

 
 from lit.llvm import llvm_config
 from lit.llvm.subst import ToolSubst
 from lit.llvm.subst import FindTool
 
 # Configuration file for the 'lit' test runner.
 
 # name: The name of this test suite.
-config.name = 'STANDALONE_OPT'
+config.name = 'STANDALONE'
 
 config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
 
 # suffixes: A list of file extensions to treat as test files.
 config.suffixes = ['.mlir']
 
 # test_source_root: The root path where tests are located.
 config.test_source_root = os.path.dirname(__file__)

 config.test_exec_root = os.path.join(config.standalone_obj_root, 'test')
 config.standalone_tools_dir = os.path.join(config.standalone_obj_root, 'bin')
 
 # Tweak the PATH to include the tools dir.
 llvm_config.with_environment('PATH', config.llvm_tools_dir, append_path=True)
 
 tool_dirs = [config.standalone_tools_dir, config.llvm_tools_dir]
 tools = [
-    'standalone-opt'
+    'standalone-opt',
+    'standalone-translate'
 ]
 
 llvm_config.add_tool_substitutions(tools, tool_dirs)

mlir/test/Examples/standalone/test.toy

-# RUN: %cmake %mlir_src_root/examples/standalone -DCMAKE_CXX_COMPILER=%host_cxx -DCMAKE_C_COMPILER=%host_cc -DMLIR_DIR=%llvm_lib_dir/cmake/mlir ; %cmake --build . --target check-standalone-opt | tee %t | FileCheck %s
+# RUN: %cmake %mlir_src_root/examples/standalone -DCMAKE_CXX_COMPILER=%host_cxx -DCMAKE_C_COMPILER=%host_cc -DMLIR_DIR=%llvm_lib_dir/cmake/mlir ; %cmake --build . --target check-standalone | tee %t | FileCheck %s
 
-# CHECK: Expected Passes: 1
+# CHECK: Expected Passes: 2
 # UNSUPPORTED: windows, android