[MLIR] [CMake] Support building MLIR standalone

[mgorny]

Summary

Add the necessary bits to CMakeLists to make it possible to configure
MLIR against installed LLVM, and build it with minimal need for LLVM
source tree. The latter is only necessary to run unittests, and if it
is missing then unittests are skipped with a warning.

This change includes the necessary changes to tests, in particular
adding some missing substitutions and defining missing variables
for lit.site.cfg.py substitution.

[stephenneuendorffer]

mlir/CMakeLists.txt

39–48

This seems like exactly the kindof boilerplate that 'every' project needs. I'd like to work refactoring this so that projects have more commonality in how they integrate with CMAKE. (I'll point you to a review along these lines)

54–57

This exists check seems awkward. I'd expect that this should actually be a configuration option?

mlir/lib/Conversion/GPUToNVVM/CMakeLists.txt

4 ↗

(On Diff #283695)

I don't understand why these include directories should be different when MLIR is built out-of-tree.

[mgorny]

mlir/CMakeLists.txt

39–48

I don't have a strong opinion on this. However, some code would have to repeated to get the 'common' stuff in anyway ;-).

54–57

I don't know if it's something worth configuring. The rough idea is that you either build with complete LLVM source tree available (and I make here a rough default for monorepo) and then gtest is available, or you don't and then it's not.

mlir/lib/Conversion/GPUToNVVM/CMakeLists.txt

4 ↗

(On Diff #283695)

Me neither. However, for some reason they're missing there and build is failing to find generated includes. I'm afraid I don't know how to debug this.

[jpienaar]

mlir/CMakeLists.txt

1–27

Could we have this line at the top still?

54–57

Could TARGET gtest not be used here too?

mlir/lib/Conversion/GPUToNVVM/CMakeLists.txt

4 ↗

(On Diff #283695)

What errors do you see?

[mgorny]

mlir/CMakeLists.txt

1–27

Sure, done that.

54–57

There's no TARGET until you add_subdirectory.

mlir/lib/Conversion/GPUToNVVM/CMakeLists.txt

4 ↗

(On Diff #283695)

It doesn't find generated .inc files (i.e. -I${CMAKE_CURRENT_BINARY_DIR} is clearly missing for some reason).

[mgorny]

Moved comment back to the top.

[mgorny]

Ping.

[stephenneuendorffer]

mlir/lib/Conversion/GPUToNVVM/CMakeLists.txt

4 ↗

(On Diff #283695)

LLVM does: CMakeLists.txt:include_directories( ${LLVM_INCLUDE_DIR} ${LLVM_MAIN_INCLUDE_DIR})
which MLIR inherits in an in-tree build

I think the correct thing to do is to have something similar:
include_directories( ${MLIR_INCLUDE_DIR} )

This should avoid including this in all of the dialects.

[isuruf]

Is there any interest in this? If so, I can rebase this on top of master and send it.

My interest here is to package MLIR for a package manager and having standalone builds makes it much easier to separately package MLIR and LLVM.

[mehdi_amini]

If you rebase, I can try to help the review if @stephenneuendorffer does not have cycles available.

[isuruf]

Rebased on top of master and fixes for new directories

[stephenneuendorffer]

mlir/lib/Conversion/GPUToNVVM/CMakeLists.txt

4 ↗

(On Diff #283695)

I still think there should be a better solution than doing this in all dialects.... maybe add it to mlir_tablegen()?

[isuruf]

Move adding CMAKE_CURRENT_BINARY_DIR to tablegen function

[stephenneuendorffer]

Looks good to me. Thanks for doing this!

[mehdi_amini]

mlir/CMakeLists.txt

140

This logic does not work for me: I suddenly get "gtest not found, unittests will not be available" in my regular build.

[isuruf]

mlir/CMakeLists.txt

140

Thanks for letting me know. Fix in https://reviews.llvm.org/D95978

[mehdi_amini]

mlir/test/CMakeLists.txt

60

Since this change, unit-tests are not built with ninja check-mlir anymore. I tried a fix in http://github.com/llvm/llvm-project/commit/09cfec62432993abde7829a88ce4793d8805ce4b ; please take a look!

mlir/CMakeLists.txt

 # MLIR project.
+
+# Check if MLIR is built as a standalone project.
+if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
+  project(mlir)
+  cmake_minimum_required(VERSION 3.13.4)
+
+  find_package(LLVM CONFIG REQUIRED)
+  set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${LLVM_CMAKE_DIR})
+  include(HandleLLVMOptions)
+  include(AddLLVM)
+  include(TableGen)
+
+  include_directories(${LLVM_INCLUDE_DIRS})
+
+  set(LLVM_MAIN_SRC_DIR ${CMAKE_SOURCE_DIR}/../llvm CACHE PATH
+      "Path to LLVM source tree")
+  set(UNITTEST_DIR ${LLVM_MAIN_SRC_DIR}/utils/unittest)
+  if(EXISTS ${UNITTEST_DIR}/googletest/include/gtest/gtest.h)
+    add_subdirectory(${UNITTEST_DIR} utils/unittest)
+  endif()
+
+  set(CMAKE_LIBRARY_OUTPUT_DIRECTORY
+    "${CMAKE_CURRENT_BINARY_DIR}/lib${LLVM_LIBDIR_SUFFIX}")
+  set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/bin")
+endif()
+
 set(MLIR_MAIN_SRC_DIR     ${CMAKE_CURRENT_SOURCE_DIR}  )
 set(MLIR_MAIN_INCLUDE_DIR ${MLIR_MAIN_SRC_DIR}/include )
 
 set(MLIR_SOURCE_DIR  ${CMAKE_CURRENT_SOURCE_DIR})
 set(MLIR_BINARY_DIR  ${CMAKE_CURRENT_BINARY_DIR})
 set(MLIR_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/include)
+set(MLIR_TOOLS_DIR   ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules")
 
 include(AddMLIR)
 
 # Forbid implicit function declaration: this may lead to subtle bugs and we
 # don't have a reason to support this.
 check_c_compiler_flag("-Werror=implicit-function-declaration" C_SUPPORTS_WERROR_IMPLICIT_FUNCTION_DECLARATION)
 append_if(C_SUPPORTS_WERROR_IMPLICIT_FUNCTION_DECLARATION "-Werror=implicit-function-declaration" CMAKE_C_FLAGS)
 
 # Installing the headers and docs needs to depend on generating any public
 # tablegen'd targets.
 # mlir-generic-headers are dialect-independent.
 add_custom_target(mlir-generic-headers)
 set_target_properties(mlir-generic-headers PROPERTIES FOLDER "Misc")
 # mlir-headers may be dialect-dependent.
 add_custom_target(mlir-headers)
 set_target_properties(mlir-headers PROPERTIES FOLDER "Misc")
 add_dependencies(mlir-headers mlir-generic-headers)
 add_custom_target(mlir-doc)
 
 # Build the CUDA conversions and run according tests if the NVPTX backend
 # is available
 if ("NVPTX" IN_LIST LLVM_TARGETS_TO_BUILD)
   set(MLIR_CUDA_CONVERSIONS_ENABLED 1)
 else()
   set(MLIR_CUDA_CONVERSIONS_ENABLED 0)
 endif()
 # TODO: we should use a config.h file like LLVM does
 add_definitions(-DMLIR_CUDA_CONVERSIONS_ENABLED=${MLIR_CUDA_CONVERSIONS_ENABLED})
 

 add_subdirectory(tools/mlir-linalg-ods-gen)
 
 add_subdirectory(include/mlir)
 add_subdirectory(lib)
 # C API needs all dialects for registration, but should be built before tests.
 add_subdirectory(lib/CAPI)
 if (MLIR_INCLUDE_TESTS)
   add_definitions(-DMLIR_INCLUDE_TESTS)
+  if (TARGET gtest)
     add_subdirectory(unittests)
+  else()
+    message(WARNING "gtest not found, unittests will not be available")
+  endif()
   add_subdirectory(test)
 endif()
 if (MLIR_INCLUDE_INTEGRATION_TESTS)
   add_definitions(-DMLIR_INCLUDE_INTEGRATION_TESTS)
   add_subdirectory(integration_test)
 endif()
 # Tools needs to come late to ensure that MLIR_ALL_LIBS is populated.
 # Generally things after this point may depend on MLIR_ALL_LIBS or libMLIR.so.

mlir/cmake/modules/AddMLIR.cmake

 function(mlir_tablegen ofn)
   tablegen(MLIR ${ARGV})
   set(TABLEGEN_OUTPUT ${TABLEGEN_OUTPUT} ${CMAKE_CURRENT_BINARY_DIR}/${ofn}
       PARENT_SCOPE)
+  include_directories(${CMAKE_CURRENT_BINARY_DIR})
 endfunction()
 
 # Declare a dialect in the include directory
 function(add_mlir_dialect dialect dialect_namespace)
   set(LLVM_TARGET_DEFINITIONS ${dialect}.td)
   mlir_tablegen(${dialect}.h.inc -gen-op-decls)
   mlir_tablegen(${dialect}.cpp.inc -gen-op-defs)
   mlir_tablegen(${dialect}Types.h.inc -gen-typedef-decls)

mlir/test/CMakeLists.txt

configure_lit_site_cfg(
   ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.py.in
   ${CMAKE_CURRENT_BINARY_DIR}/Unit/lit.site.cfg.py
   MAIN_CONFIG
   ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.cfg.py
   )
 
 set(MLIR_TEST_DEPENDS
   FileCheck count not
-  MLIRUnitTests
   mlir-capi-ir-test
   mlir-capi-pass-test
   mlir-cpu-runner
   mlir-edsc-builder-api-test
   mlir-linalg-ods-gen
   mlir-opt
   mlir-reduce
   mlir-sdbm-api-test
   mlir-tblgen
   mlir-translate
   mlir_runner_utils
   mlir_c_runner_utils
   mlir_async_runtime
   )
 
+if(TARGET gtest)
+  list(APPEND MLIR_TEST_DEPENDS MLIRUnitTests)
+endif()
+
 if(LLVM_BUILD_EXAMPLES)
   list(APPEND MLIR_TEST_DEPENDS
     toyc-ch1
     toyc-ch2
     toyc-ch3
     toyc-ch4
     toyc-ch5
     toyc-ch6

mlir/test/lit.cfg.py

 tool_dirs = [config.mlir_tools_dir, config.llvm_tools_dir]
 tools = [
     'mlir-opt',
     'mlir-tblgen',
     'mlir-translate',
     'mlir-capi-ir-test',
     'mlir-capi-pass-test',
     'mlir-edsc-builder-api-test',
+    'mlir-cpu-runner',
+    'mlir-linalg-ods-gen',
+    'mlir-reduce',
+    'mlir-sdbm-api-test',
 ]
 
 # The following tools are optional
 tools.extend([
     ToolSubst('%PYTHON', config.python_executable, unresolved='ignore'),
     ToolSubst('toy-ch1', unresolved='ignore'),
     ToolSubst('toy-ch2', unresolved='ignore'),
     ToolSubst('toy-ch3', unresolved='ignore'),