[MLIR] [CMake] Support building MLIR standalone

[mgorny]

Summary

Add the necessary bits to CMakeLists to make it possible to configure
MLIR against installed LLVM, and build it with minimal need for LLVM
source tree. The latter is only necessary to run unittests, and if it
is missing then unittests are skipped with a warning.

This change includes the necessary changes to tests, in particular
adding some missing substitutions and defining missing variables
for lit.site.cfg.py substitution.

[stephenneuendorffer]

mlir/CMakeLists.txt

29–38

This seems like exactly the kindof boilerplate that 'every' project needs. I'd like to work refactoring this so that projects have more commonality in how they integrate with CMAKE. (I'll point you to a review along these lines)

44–47

This exists check seems awkward. I'd expect that this should actually be a configuration option?

mlir/lib/Conversion/GPUToNVVM/CMakeLists.txt

4

I don't understand why these include directories should be different when MLIR is built out-of-tree.

[mgorny]

mlir/CMakeLists.txt

29–38

I don't have a strong opinion on this. However, some code would have to repeated to get the 'common' stuff in anyway ;-).

44–47

I don't know if it's something worth configuring. The rough idea is that you either build with complete LLVM source tree available (and I make here a rough default for monorepo) and then gtest is available, or you don't and then it's not.

mlir/lib/Conversion/GPUToNVVM/CMakeLists.txt

4

Me neither. However, for some reason they're missing there and build is failing to find generated includes. I'm afraid I don't know how to debug this.

[jpienaar]

mlir/CMakeLists.txt

1–27

Could we have this line at the top still?

44–47

Could TARGET gtest not be used here too?

mlir/lib/Conversion/GPUToNVVM/CMakeLists.txt

4

What errors do you see?

[mgorny]

mlir/CMakeLists.txt

1–27

Sure, done that.

44–47

There's no TARGET until you add_subdirectory.

mlir/lib/Conversion/GPUToNVVM/CMakeLists.txt

4

It doesn't find generated .inc files (i.e. -I${CMAKE_CURRENT_BINARY_DIR} is clearly missing for some reason).

[mgorny]

Moved comment back to the top.

[mgorny]

Ping.

[stephenneuendorffer]

mlir/lib/Conversion/GPUToNVVM/CMakeLists.txt

4

LLVM does: CMakeLists.txt:include_directories( ${LLVM_INCLUDE_DIR} ${LLVM_MAIN_INCLUDE_DIR})
which MLIR inherits in an in-tree build

I think the correct thing to do is to have something similar:
include_directories( ${MLIR_INCLUDE_DIR} )

This should avoid including this in all of the dialects.

[isuruf]

Is there any interest in this? If so, I can rebase this on top of master and send it.

My interest here is to package MLIR for a package manager and having standalone builds makes it much easier to separately package MLIR and LLVM.

[mehdi_amini]

If you rebase, I can try to help the review if @stephenneuendorffer does not have cycles available.

[isuruf]

Rebased on top of master and fixes for new directories

[stephenneuendorffer]

mlir/lib/Conversion/GPUToNVVM/CMakeLists.txt

4

I still think there should be a better solution than doing this in all dialects.... maybe add it to mlir_tablegen()?

[isuruf]

Move adding CMAKE_CURRENT_BINARY_DIR to tablegen function

[stephenneuendorffer]

Looks good to me. Thanks for doing this!

[mehdi_amini]

mlir/CMakeLists.txt

131

This logic does not work for me: I suddenly get "gtest not found, unittests will not be available" in my regular build.

[isuruf]

mlir/CMakeLists.txt

131

Thanks for letting me know. Fix in https://reviews.llvm.org/D95978

[mehdi_amini]

mlir/test/CMakeLists.txt

61

Since this change, unit-tests are not built with ninja check-mlir anymore. I tried a fix in http://github.com/llvm/llvm-project/commit/09cfec62432993abde7829a88ce4793d8805ce4b ; please take a look!

mlir/CMakeLists.txt

 # MLIR project.
+
+# Check if MLIR is built as a standalone project.
+if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
+  project(mlir)
+  cmake_minimum_required(VERSION 3.13.4)
+
+  find_package(LLVM CONFIG REQUIRED)
+  set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${LLVM_CMAKE_DIR})
+  include(HandleLLVMOptions)
+  include(AddLLVM)
+  include(TableGen)
+
+  include_directories(${LLVM_INCLUDE_DIRS})
+
+  set(LLVM_MAIN_SRC_DIR ${CMAKE_SOURCE_DIR}/../llvm CACHE PATH
+      "Path to LLVM source tree")
+  set(UNITTEST_DIR ${LLVM_MAIN_SRC_DIR}/utils/unittest)
+  if(EXISTS ${UNITTEST_DIR}/googletest/include/gtest/gtest.h)
+    add_subdirectory(${UNITTEST_DIR} utils/unittest)
+  endif()
+
+  set(CMAKE_LIBRARY_OUTPUT_DIRECTORY
+    "${CMAKE_CURRENT_BINARY_DIR}/lib${LLVM_LIBDIR_SUFFIX}")
+  set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/bin")
+endif()
+
 set(MLIR_MAIN_SRC_DIR     ${CMAKE_CURRENT_SOURCE_DIR}  )
 set(MLIR_MAIN_INCLUDE_DIR ${MLIR_MAIN_SRC_DIR}/include )
 
 set(MLIR_SOURCE_DIR  ${CMAKE_CURRENT_SOURCE_DIR})
 set(MLIR_BINARY_DIR  ${CMAKE_CURRENT_BINARY_DIR})
 set(MLIR_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/include)
+set(MLIR_TOOLS_DIR   ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules")
 
 include(AddMLIR)
 
 # Installing the headers and docs needs to depend on generating any public
 # tablegen'd targets.
 # mlir-generic-headers are dialect-independent.
 add_custom_target(mlir-generic-headers)
 set_target_properties(mlir-generic-headers PROPERTIES FOLDER "Misc")
 # mlir-headers may be dialect-dependent.
 add_custom_target(mlir-headers)
 set_target_properties(mlir-headers PROPERTIES FOLDER "Misc")
 add_dependencies(mlir-headers mlir-generic-headers)
 add_custom_target(mlir-doc)
 
 # Build the CUDA conversions and run according tests if the NVPTX backend
 # is available
 if ("NVPTX" IN_LIST LLVM_TARGETS_TO_BUILD)
   set(MLIR_CUDA_CONVERSIONS_ENABLED 1)
 else()

 add_subdirectory(tools/mlir-tblgen)
 
 add_subdirectory(include/mlir)
 add_subdirectory(lib)
 # C API needs all dialects for registration, but should be built before tests.
 add_subdirectory(lib/CAPI)
 if (MLIR_INCLUDE_TESTS)
   add_definitions(-DMLIR_INCLUDE_TESTS)
+  if (TARGET gtest)
     add_subdirectory(unittests)
+  else()
+    message(WARNING "gtest not found, unittests will not be available")
+  endif()
   add_subdirectory(test)
 endif()
 if (MLIR_INCLUDE_INTEGRATION_TESTS)
   add_definitions(-DMLIR_INCLUDE_INTEGRATION_TESTS)
   add_subdirectory(integration_test)
 endif()
 # Tools needs to come late to ensure that MLIR_ALL_LIBS is populated.
 # Generally things after this point may depend on MLIR_ALL_LIBS or libMLIR.so.

mlir/lib/Conversion/GPUToNVVM/CMakeLists.txt

 set(LLVM_TARGET_DEFINITIONS GPUToNVVM.td)
 mlir_tablegen(GPUToNVVM.cpp.inc -gen-rewriters)
 add_public_tablegen_target(MLIRGPUToNVVMIncGen)
+include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
 add_mlir_conversion_library(MLIRGPUToNVVMTransforms
   LowerGpuOpsToNVVMOps.cpp
 
   DEPENDS
   MLIRConversionPassIncGen
   MLIRGPUToNVVMIncGen
 
   LINK_LIBS PUBLIC
   MLIRGPU
   MLIRLLVMIR
   MLIRNVVMIR
   MLIRPass
   MLIRStandardToLLVM
   MLIRTransformUtils
   )

mlir/lib/Conversion/GPUToROCDL/CMakeLists.txt

 set(LLVM_TARGET_DEFINITIONS GPUToROCDL.td)
 mlir_tablegen(GPUToROCDL.cpp.inc -gen-rewriters)
 add_public_tablegen_target(MLIRGPUToROCDLIncGen)
+include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
 add_mlir_conversion_library(MLIRGPUToROCDLTransforms
   LowerGpuOpsToROCDLOps.cpp
 
   DEPENDS
   MLIRConversionPassIncGen
   MLIRGPUToROCDLIncGen
 
   LINK_LIBS PUBLIC
   MLIRGPU
   MLIRLLVMIR
   MLIRROCDLIR
   MLIRPass
   MLIRStandardToLLVM
   MLIRVectorToROCDL
   )

mlir/lib/Conversion/GPUToSPIRV/CMakeLists.txt

 set(LLVM_TARGET_DEFINITIONS GPUToSPIRV.td)
 mlir_tablegen(GPUToSPIRV.cpp.inc -gen-rewriters)
 add_public_tablegen_target(MLIRGPUToSPIRVIncGen)
+include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
 add_mlir_conversion_library(MLIRGPUToSPIRVTransforms
   ConvertGPUToSPIRV.cpp
   ConvertGPUToSPIRVPass.cpp
 
   DEPENDS
   MLIRConversionPassIncGen
   MLIRGPUToSPIRVIncGen

mlir/lib/Dialect/SPIRV/CMakeLists.txt

 
 set(LLVM_TARGET_DEFINITIONS SPIRVCanonicalization.td)
 mlir_tablegen(SPIRVCanonicalization.inc -gen-rewriters)
 add_public_tablegen_target(MLIRSPIRVCanonicalizationIncGen)
+include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
 add_mlir_dialect_library(MLIRSPIRV
   LayoutUtils.cpp
   SPIRVAttributes.cpp
   SPIRVCanonicalization.cpp
   SPIRVDialect.cpp
   SPIRVOps.cpp
   SPIRVLowering.cpp

mlir/lib/Dialect/Shape/IR/CMakeLists.txt

 set(LLVM_TARGET_DEFINITIONS ShapeCanonicalization.td)
 mlir_tablegen(ShapeCanonicalization.inc -gen-rewriters)
 add_public_tablegen_target(MLIRShapeCanonicalizationIncGen)
+include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
 add_mlir_dialect_library(MLIRShape
   Shape.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/Shape
 
   DEPENDS

mlir/test/CMakeLists.txt

configure_lit_site_cfg(
   ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.py.in
   ${CMAKE_CURRENT_BINARY_DIR}/Unit/lit.site.cfg.py
   MAIN_CONFIG
   ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.cfg.py
   )
 
 set(MLIR_TEST_DEPENDS
   FileCheck count not
-  MLIRUnitTests
   mlir-capi-ir-test
   mlir-cpu-runner
   mlir-edsc-builder-api-test
   mlir-linalg-ods-gen
   mlir-opt
   mlir-reduce
   mlir-sdbm-api-test
   mlir-tblgen
   mlir-translate
   mlir_test_cblas
   mlir_test_cblas_interface
   mlir_runner_utils
   mlir_c_runner_utils
   )
+if(TARGET gtest)
+  list(APPEND MLIR_TEST_DEPENDS MLIRUnitTests)
+endif()
 
 if(LLVM_BUILD_EXAMPLES)
   list(APPEND MLIR_TEST_DEPENDS
     toyc-ch1
     toyc-ch2
     toyc-ch3
     toyc-ch4
     toyc-ch5
     toyc-ch6
     toyc-ch7

mlir/test/lib/Dialect/Test/CMakeLists.txt

 set(LLVM_OPTIONAL_SOURCES
   TestDialect.cpp
   TestPatterns.cpp
 )
+include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
 set(LLVM_TARGET_DEFINITIONS TestInterfaces.td)
 mlir_tablegen(TestTypeInterfaces.h.inc -gen-type-interface-decls)
 mlir_tablegen(TestTypeInterfaces.cpp.inc -gen-type-interface-defs)
 add_public_tablegen_target(MLIRTestInterfaceIncGen)
 
 set(LLVM_TARGET_DEFINITIONS TestOps.td)
 mlir_tablegen(TestOps.h.inc -gen-op-decls)

mlir/test/lit.cfg.py

 
 tool_dirs = [config.mlir_tools_dir, config.llvm_tools_dir]
 tools = [
     'mlir-opt',
     'mlir-tblgen',
     'mlir-translate',
     'mlir-capi-ir-test',
     'mlir-edsc-builder-api-test',
+    'mlir-cpu-runner',
+    'mlir-linalg-ods-gen',
+    'mlir-reduce',
+    'mlir-sdbm-api-test',
 ]
 
 # The following tools are optional
 tools.extend([
     ToolSubst('%PYTHON', config.python_executable),
     ToolSubst('toy-ch1', unresolved='ignore'),
     ToolSubst('toy-ch2', unresolved='ignore'),
     ToolSubst('toy-ch3', unresolved='ignore'),

mlir/unittests/TableGen/CMakeLists.txt

 )
 
 add_dependencies(MLIRTableGenTests MLIRTableGenEnumsIncGen)
 add_dependencies(MLIRTableGenTests MLIRTableGenStructAttrIncGen)
 add_dependencies(MLIRTableGenTests MLIRTestDialect)
 
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../test/lib/Dialect/Test)
 include_directories(${CMAKE_CURRENT_BINARY_DIR}/../../test/lib/Dialect/Test)
+include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
 target_link_libraries(MLIRTableGenTests
   PRIVATE MLIRTableGen MLIRIR
   PUBLIC MLIRTestDialect
 )