Make paths in generated llvm-lit relative as well.

[thakis]

Summary

This builds on top of D77184. With this, I can rename my build directory
to a different name and bin/llvm-lit ../llvm-project/clang/test ../llvm-project/llvm/test
still succeeds.

I haven't tried copying the build dir to a different machine to run tests there yet,
but I tried something like it a few months ago and it worked back then.

Changes:

• Make configure_lit_site_cfg() store the main / generated config pair interleaved in the LLVM_LIT_CONFIG_FILES list and postpone converting it to python code to llvm-lit's CMakeList.
• Pull the relpath code into a new function make_paths_relative() and call that in llvm-lit's CMakeList, prior to converting the list to python code.
• Pull the path() function into a variable and use that in llvm-lit's CMakeList too.

[hans]

Seems okay to me.

llvm/cmake/modules/AddLLVM.cmake

1430

Would it be possible to not lower-case paths in the first place?

[thakis]

Thanks!

llvm/cmake/modules/AddLLVM.cmake

1430

Yes, I'd like to try to do that next. There's a bit of history here (git log -Snormcase llvm/utils/lit) so maybe it's not super easy. OTOH the original workaround was for a fairly old MSVC, so maybe it's better now.

[broadwaylamb]

Looks like this broke some buildbots:

• http://lab.llvm.org:8011/builders/sanitizer-x86_64-linux-autoconf/builds/49184
• http://lab.llvm.org:8011/builders/llvm-clang-win-x-armv7l/builds/6614
• http://lab.llvm.org:8011/builders/llvm-clang-win-x-aarch64/builds/6654

[thakis]

Fixed the first one in 2e1afe5232674cab6edf1efefd0c8c88fb63a594

The other 2 look kind of unrelated; are you sure they're due to this change?

[broadwaylamb]

In D77496#1964102, @thakis wrote:

Fixed the first one in 2e1afe5232674cab6edf1efefd0c8c88fb63a594

Thanks!

The other 2 look kind of unrelated; are you sure they're due to this change?

Pretty sure. The error message in CMake is the same.

P. S. And I can see they're green again after your fix :-)

[thakis]

Pretty sure. The error message in CMake is the same.

P. S. And I can see they're green again after your fix :-)

Glad to hear the change fixed the other two bots too :)

llvm/cmake/modules/AddLLVM.cmake

macro(set_llvm_build_mode)
   # Configuration-time: See Unit/lit.site.cfg.in
   if (CMAKE_CFG_INTDIR STREQUAL ".")
     set(LLVM_BUILD_MODE ".")
   else ()
     set(LLVM_BUILD_MODE "%(build_mode)s")
   endif ()
 endmacro()
 
+# Takes a list of path names in pathlist and a base directory, and returns
+# a list of paths relative to the base directory in out_pathlist.
+# Paths that are on a different drive than the basedir (on Windows) or that
+# contain symlinks are returned absolute.
+# Use with LLVM_LIT_PATH_FUNCTION below.
+function(make_paths_relative out_pathlist basedir pathlist)
+  # Passing ARG_PATH_VALUES as-is to execute_process() makes cmake strip
+  # empty list entries. So escape the ;s in the list and do the splitting
+  # outselves. cmake has no relpath function, so use Python for that.
+  string(REPLACE ";" "\\;" pathlist_escaped "${pathlist}")
+  execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c" "\n
+import os, sys\n
+base = sys.argv[1]
+def haslink(p):\n
+    if not p or p == os.path.dirname(p): return False\n
+    return os.path.islink(p) or haslink(os.path.dirname(p))\n
+def relpath(p):\n
+    if not p: return ''\n
+    if os.path.splitdrive(p)[0] != os.path.splitdrive(base)[0]: return p\n
+    if haslink(p) or haslink(base): return p\n
+    return os.path.relpath(p, base).replace(os.sep, '/')\n
+sys.stdout.write(';'.join(relpath(p) for p in sys.argv[2].split(';')))"
+    ${basedir}
+    ${pathlist_escaped}
+    OUTPUT_VARIABLE pathlist_relative)
+  set(${out_pathlist} "${pathlist_relative}" PARENT_SCOPE)
+endfunction()
+
+# Converts a file that's relative to the current python file to an absolute
+# path. Since this uses __file__, it has to be emitted into python files that
+# use it and can't be in a lit module. Use with make_paths_relative().
+string(CONCAT LLVM_LIT_PATH_FUNCTION
+  # Lit converts config paths to lower case in discovery.py, before
+  # loading the config. This causes __file__ to be all lower-case (including
+  # the drive letter), but several clang tests pass -include %s and a
+  # clang warning checks that passed case matches on-disk cache. So it's
+  # important that this restores the on-disk case of the prefix.
+  "# Allow generated file to be relocatable.\n"
+  "def path(p):\n"
+  "  if not p: return ''\n"
+  "  p = os.path.join(os.path.dirname(os.path.abspath(__file__)), p)\n"
+  "  p = os.path.normpath(p).replace(os.sep, '/')\n"
+  "  if os.name == 'nt' and os.path.isabs(p): return p[0].upper() + p[1:]\n"
+  "  return p\n"
+  )
+
 # This function provides an automatic way to 'configure'-like generate a file
 # based on a set of common and custom variables, specifically targeting the
 # variables needed for the 'lit.site.cfg' files. This function bundles the
 # common variables that any Lit instance is likely to need, and custom
 # variables can be passed in.
 # The keyword PATHS is followed by a list of cmake variable names that are
 # mentioned as `path("@varname@")` in the lit.cfg.py.in file. Variables in that
 # list are treated as paths that are relative to the directory the generated
function(configure_lit_site_cfg site_in site_out)
 
   set(HOST_OS ${CMAKE_SYSTEM_NAME})
   set(HOST_ARCH ${CMAKE_SYSTEM_PROCESSOR})
 
   set(HOST_CC "${CMAKE_C_COMPILER} ${CMAKE_C_COMPILER_ARG1}")
   set(HOST_CXX "${CMAKE_CXX_COMPILER} ${CMAKE_CXX_COMPILER_ARG1}")
   set(HOST_LDFLAGS "${CMAKE_EXE_LINKER_FLAGS}")
 
-  set(LIT_SITE_CFG_IN_HEADER  "# Autogenerated from ${site_in}\n# Do not edit!")
-
-  # Lit converts config paths to lower case in discovery.py, before
-  # loading the config. This causes __file__ to be all lower-case (including
-  # the drive letter), but several clang tests pass -include %s and a
-  # clang warning checks that passed case matches on-disk cache. So it's
-  # important that this restores the on-disk case of the prefix.
-  string(CONCAT LIT_SITE_CFG_IN_HEADER "${LIT_SITE_CFG_IN_HEADER}\n\n"
-    "# Allow generated lit.site.cfg.py to be relocatable.\n"
-    "def path(p):\n"
-    "  if not p: return ''\n"
-    "  p = os.path.normpath(os.path.join(os.path.dirname(__file__), p)).replace(os.sep, '/')\n"
-    "  if os.name == 'nt' and os.path.isabs(p): return p[0].upper() + p[1:]\n"
-    "  return p\n"
+  string(CONCAT LIT_SITE_CFG_IN_HEADER
+    "# Autogenerated from ${site_in}\n# Do not edit!\n\n"
+    "${LLVM_LIT_PATH_FUNCTION}"
     )
 
   # Override config_target_triple (and the env)
   if(LLVM_TARGET_TRIPLE_ENV)
     # This is expanded into the heading.
     string(CONCAT LIT_SITE_CFG_IN_HEADER "${LIT_SITE_CFG_IN_HEADER}"
       "import os\n"
       "target_env = \"${LLVM_TARGET_TRIPLE_ENV}\"\n"
if (ARG_PATHS)
     # so start the list with a dummy element and drop it, to make sure that
     # even empty values make it into the values list.
     set(ARG_PATH_VALUES "dummy")
     foreach(path ${ARG_PATHS})
       list(APPEND ARG_PATH_VALUES "${${path}}")
     endforeach()
     list(REMOVE_AT ARG_PATH_VALUES 0)
 
-    # Compute paths relative to the directory containing output lit.site.cfg.py.
-    # Passing ARG_PATH_VALUES as-is to execute_process() makes cmake strip
-    # empty list entries. So escape the ;s in the list and do the splitting
-    # outselves. cmake has no relpath function, so use Python for that.
-    string(REPLACE ";" "\\;" ARG_PATH_VALUES_ESCAPED "${ARG_PATH_VALUES}")
     get_filename_component(OUTPUT_DIR ${site_out} DIRECTORY)
-    execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c" "\n
-import os, sys\n
-base = sys.argv[1]
-def haslink(p):\n
-    if not p or p == os.path.dirname(p): return False\n
-    return os.path.islink(p) or haslink(os.path.dirname(p))\n
-def relpath(p):\n
-    if not p: return ''\n
-    if os.path.splitdrive(p)[0] != os.path.splitdrive(base)[0]: return p\n
-    if haslink(p) or haslink(base): return p\n
-    return os.path.relpath(p, base).replace(os.sep, '/')\n
-sys.stdout.write(';'.join(relpath(p) for p in sys.argv[2].split(';')))"
-      ${OUTPUT_DIR}
-      ${ARG_PATH_VALUES_ESCAPED}
-      OUTPUT_VARIABLE ARG_PATH_VALUES_RELATIVE)
+    make_paths_relative(
+        ARG_PATH_VALUES_RELATIVE "${OUTPUT_DIR}" "${ARG_PATH_VALUES}")
 
     list(LENGTH ARG_PATHS len_paths)
     list(LENGTH ARG_PATH_VALUES len_path_values)
     list(LENGTH ARG_PATH_VALUES_RELATIVE len_path_value_rels)
     if ((NOT ${len_paths} EQUAL ${len_path_values}) OR
         (NOT ${len_paths} EQUAL ${len_path_value_rels}))
       message(SEND_ERROR "PATHS lengths got confused")
     endif()
 
     # Transform variables mentioned in ARG_PATHS to relative paths for
     # the configure_file() call. Variables are copied to subscopeds by cmake,
     # so this only modifies the local copy of the variables.
     math(EXPR arg_path_limit "${len_paths} - 1")
     foreach(i RANGE ${arg_path_limit})
       list(GET ARG_PATHS ${i} val1)
       list(GET ARG_PATH_VALUES_RELATIVE ${i} val2)
       set(${val1} ${val2})
     endforeach()
   endif()
 
   configure_file(${site_in} ${site_out} @ONLY)
 
   if (EXISTS "${ARG_MAIN_CONFIG}")
-    set(PYTHON_STATEMENT "map_config('${ARG_MAIN_CONFIG}', '${site_out}')")
-    get_property(LLVM_LIT_CONFIG_MAP GLOBAL PROPERTY LLVM_LIT_CONFIG_MAP)
-    set(LLVM_LIT_CONFIG_MAP "${LLVM_LIT_CONFIG_MAP}\n${PYTHON_STATEMENT}")
-    set_property(GLOBAL PROPERTY LLVM_LIT_CONFIG_MAP ${LLVM_LIT_CONFIG_MAP})
+    # Remember main config / generated site config for llvm-lit.in.
+    get_property(LLVM_LIT_CONFIG_FILES GLOBAL PROPERTY LLVM_LIT_CONFIG_FILES)
+    list(APPEND LLVM_LIT_CONFIG_FILES "${ARG_MAIN_CONFIG}" "${site_out}")
+    set_property(GLOBAL PROPERTY LLVM_LIT_CONFIG_FILES ${LLVM_LIT_CONFIG_FILES})
   endif()
 endfunction()
 
 function(dump_all_cmake_variables)
   get_cmake_property(_variableNames VARIABLES)
   foreach (_variableName ${_variableNames})
     message(STATUS "${_variableName}=${${_variableName}}")
   endforeach()

llvm/utils/llvm-lit/CMakeLists.txt

-get_property(LLVM_LIT_CONFIG_MAP GLOBAL PROPERTY LLVM_LIT_CONFIG_MAP)
+get_property(LLVM_LIT_CONFIG_FILES GLOBAL PROPERTY LLVM_LIT_CONFIG_FILES)
+list(LENGTH LLVM_LIT_CONFIG_FILES file_len)
+math(EXPR file_last "${file_len} - 1")
 
 get_llvm_lit_path(LIT_BASE_DIR LIT_FILE_NAME)
 
+# LLVM_LIT_CONFIG_FILES contains interleaved main config (in the source tree)
+# and site config (in the build tree) pairs. Make them relative to
+# llvm-lit and then convert them to map_config() calls.
+if("${CMAKE_CFG_INTDIR}" STREQUAL ".")
+  make_paths_relative(
+      LLVM_LIT_CONFIG_FILES "${LIT_BASE_DIR}" "${LLVM_LIT_CONFIG_FILES}")
+endif()
+
+set(LLVM_LIT_CONFIG_MAP "${LLVM_LIT_PATH_FUNCTION}\n")
+foreach(i RANGE 0 ${file_last} 2)
+  list(GET LLVM_LIT_CONFIG_FILES ${i} main_config)
+  math(EXPR i1 "${i} + 1")
+  list(GET LLVM_LIT_CONFIG_FILES ${i1} site_out)
+  set(map "map_config(path('${main_config}'), path('${site_out}'))")
+  set(LLVM_LIT_CONFIG_MAP "${LLVM_LIT_CONFIG_MAP}\n${map}")
+endforeach()
+
 set(LLVM_SOURCE_DIR ${LLVM_MAIN_SRC_DIR})
 
 if(NOT "${CMAKE_CFG_INTDIR}" STREQUAL ".")
   foreach(BUILD_MODE ${CMAKE_CONFIGURATION_TYPES})
     string(REPLACE ${CMAKE_CFG_INTDIR} ${BUILD_MODE} bi ${LIT_BASE_DIR})
     set(bi "${bi}/${LIT_FILE_NAME}")
     configure_file(
       llvm-lit.in