[CMake] Attempt to merge runtimes and projects compilation databases

[jhuber6]

Summary

When building a project in a runtime mode, the compilation database is a
separate CMake invocation. So its compile_commands.json file will be
placed elsewhere in the runtimes/runtime-bins directory. This is
somewhat annoying for ongoing development when a runtimes build is
necessary. This patch adds some CMake magic to merge the two files.

[jhuber6]

Unfortunately this does not work in every case I'd like. When the main CMake gets updated it will overwrite the compile_commands.json without touching the one in runtimes/. I can't make the command depend on the main compile_commands.json because it would create a circular dependency as it is the file we are writing to. Let me know if there's a hook I can use to get past that.

[jdoerfert]

llvm/utils/concat-compile-commands.py

19

JSON

21

Is it really created below?

[jhuber6]

Fix help message that wasn't updated correctly.

[MaskRay]

Thanks for the patch. This is very useful! Though I am not good at CMake to review this...

[jhuber6]

Fix leftover print. I'm still wondering if there's a good way to get the CMake to properly keep this updated. Basically, some way to make the dependency of the command depend on compile_commnads.json but not when we write it. Maybe a really hacky way to do it would be to cancel the command if the input is too new.

[tianshilei1992]

I wonder if clang is not built yet, what will be used (as for compiler) in the runtimes compilation database?

[jhuber6]

In D147441#4338295, @tianshilei1992 wrote:

I wonder if clang is not built yet, what will be used (as for compiler) in the runtimes compilation database?

I thought the whole point of the runtimes build was to manually set the compiler to the newly built clang and re-run CMake in a separate directory. The main problem I had with this patch was the I can't detect changes to the base compile_commnads because we write to it when we update it. I'm not sure if there's a good way to tell CMake "This should be updated if anyone but this function changes the file".

[jdoerfert]

This works now? What are the leftover problems?

[jhuber6]

In D147441#4342803, @jdoerfert wrote:

This works now? What are the leftover problems?

As I mentioned above. If someone changes something in CMake it will regenerate the compile_commands.json list. So ideally we'd put a dependency on that file for this action so we auto-merge them. The problem is that this dependency will then fire on the merging itself because we modify the file. So, in its current state this patch will merge the files automatically once at build time, but then won't do it with successive builds.

llvm/runtimes/CMakeLists.txt

set(RUNTIMES_TEST_DEPENDS
         split-file
       )
     foreach(target ${test_targets} ${SUB_CHECK_TARGETS})
       add_dependencies(${target} ${RUNTIMES_TEST_DEPENDS})
     endforeach()
 
     set_property(GLOBAL APPEND PROPERTY LLVM_ALL_ADDITIONAL_TEST_TARGETS runtimes ${RUNTIMES_TEST_DEPENDS})
   endif()
+
+  # Attempt to merge the runtimes' compile_commands.json file into the main
+  # file.
+  if(CMAKE_EXPORT_COMPILE_COMMANDS AND TARGET runtimes AND TARGET runtimes-configure)
+    set(files_to_concat
+        ${LLVM_BINARY_DIR}/compile_commands.json
+        ${LLVM_BINARY_DIR}/runtimes/runtimes-bins/compile_commands.json
+    )
+    set(output_file ${LLVM_BINARY_DIR}/compile_commands.json)
+    add_custom_command(
+      OUTPUT ${output_file}
+      COMMAND "${Python3_EXECUTABLE}" ${LLVM_MAIN_SRC_DIR}/utils/concat-compile-commands.py ${files_to_concat} -o ${output_file}
+      COMMENT "Merging the LLVM runtimes 'compile_commands.json' file"
+      DEPENDS runtimes-configure ${LLVM_BINARY_DIR}/runtimes/runtimes-bins/compile_commands.json
+      VERBATIM
+    )
+    add_custom_target(merge-compile-commands DEPENDS ${output_file})
+    add_dependencies(runtimes merge-compile-commands)
+  endif()
 endif()

llvm/utils/concat-compile-commands.py

+#!/usr/bin/env python
+
+import argparse
+import json
+
+desc = '''Concatenate JSON files generated by CMake tools.'''
+
+def main():
+    parser = argparse.ArgumentParser(description=desc)
+    parser.add_argument(
+        'json_files',
+        nargs='+',
+        help='list of JSON files to concatenate.')
+    parser.add_argument(
+        '--output',
+        '-o',
+        required=True,
+        help='Filename to output the merged JSON file to.')
+    
+    data = []
+    args = parser.parse_args()
+    for json_file in args.json_files:
+        with open(json_file) as file:
+            data.extend(json.load(file))
+
+    with open(args.output, 'w') as file:
+        json.dump(data, file, indent=2)
+
+if __name__ == '__main__':
+    main()