[asan] Add support for running lit tests in the iOS Simulator

[kubamracek]

Summary

Hi, this patch adds a basic support for running the ASan lit test suite against an iOS Simulator. This is done by generating more lit.site.cfg configurations into subdirectories such as IOSSimI386Config and IOSSimX86_64Config. These test suites are not added into "check-all" or into "check-asan" (and I don't even plan to add them there), and they have to be run manually, plus you need to specify which device to use (you usually have plenty of simulators installed):

$ SANITIZER_IOSSIM_TEST_DEVICE_IDENTIFIER=BBE44C1C-8AAA-4000-8D06-91C89ED58172
$ ./bin/llvm-lit ./projects/compiler-rt/test/asan/IOSSimI386Config

There will be plenty of failing tests, but I'll work on those later.

[filcab]

We should probably mention what the SANITIZER_IOS_TEST_DEVICE_IDENTIFIER and SANITIZER_IOSSIM_TEST_DEVICE_IDENTIFIER variables should contain, and how to get those values.

test/asan/lit.cfg

110 ↗

(On Diff #93400)

Please split this cleanup into a separate change (or just commit it separately)

127 ↗

(On Diff #93400)

Don't add the %run substitution only to remove it later, please.
It will make it harder to figure out what the actual %run substitution is. Adding a if config.host_os != 'Darwin' or not config.iossim should be enough.

test/asan/lit.site.cfg.in

11 ↗

(On Diff #93400)

Please use pythonize_bool and the _PYBOOL versions of these.

[filcab]

test/asan/lit.cfg

138 ↗

(On Diff #93400)

Please choose only one of these unless there's a very good reason to have different ones.

Actually: Please don't add features unless you need them (and add a test-case showing they're needed when you add them).

[kubamracek]

Addressing review comments. Removed things that aren't immediately used. Using pythonize_bool.

[kubamracek]

In D31477#714011, @filcab wrote:

We should probably mention what the SANITIZER_IOS_TEST_DEVICE_IDENTIFIER and SANITIZER_IOSSIM_TEST_DEVICE_IDENTIFIER variables should contain, and how to get those values.

Each iOS device and simulator has a UUID, you can find it in Xcode in "Devices" window.

[kubamracek]

Is this okay to land?

compiler-rt/trunk/cmake/Modules/AddCompilerRT.cmake

foreach(libname ${libnames})
     if(${type} STREQUAL "SHARED")
       if(LIB_LINK_LIBS)
         target_link_libraries(${libname} ${LIB_LINK_LIBS})
       endif()
       if(WIN32 AND NOT CYGWIN AND NOT MINGW)
         set_target_properties(${libname} PROPERTIES IMPORT_PREFIX "")
         set_target_properties(${libname} PROPERTIES IMPORT_SUFFIX ".lib")
       endif()
+      if(APPLE)
+        # Ad-hoc sign the dylibs
+        add_custom_command(TARGET ${libname}
+          POST_BUILD  
+          COMMAND codesign --sign - $<TARGET_FILE:${libname}>
+          WORKING_DIRECTORY ${COMPILER_RT_LIBRARY_OUTPUT_DIR}
+        )
+      endif()
     endif()
     install(TARGETS ${libname}
       ARCHIVE DESTINATION ${COMPILER_RT_LIBRARY_INSTALL_DIR}
               ${COMPONENT_OPTION}
       LIBRARY DESTINATION ${COMPILER_RT_LIBRARY_INSTALL_DIR}
               ${COMPONENT_OPTION}
       RUNTIME DESTINATION ${COMPILER_RT_LIBRARY_INSTALL_DIR}
               ${COMPONENT_OPTION})

compiler-rt/trunk/test/asan/CMakeLists.txt

 endif()
 
 foreach(arch ${ASAN_TEST_ARCH})
   if(ANDROID)
     set(ASAN_TEST_TARGET_ARCH ${arch}-android)
   else()
     set(ASAN_TEST_TARGET_ARCH ${arch})
   endif()
+
+  set(ASAN_TEST_IOS "0")
+  pythonize_bool(ASAN_TEST_IOS)
+  set(ASAN_TEST_IOSSIM "0")
+  pythonize_bool(ASAN_TEST_IOSSIM)
+
   string(TOLOWER "-${arch}-${OS_NAME}" ASAN_TEST_CONFIG_SUFFIX)
   get_bits_for_arch(${arch} ASAN_TEST_BITS)
   get_test_cc_for_arch(${arch} ASAN_TEST_TARGET_CC ASAN_TEST_TARGET_CFLAGS)
   if(ANDROID)
     set(ASAN_TEST_DYNAMIC True)
   else()
     set(ASAN_TEST_DYNAMIC False)
   endif()
if(COMPILER_RT_ASAN_HAS_STATIC_RUNTIME)
     configure_lit_site_cfg(
       ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.in
       ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg)
     list(APPEND ASAN_DYNAMIC_TESTSUITES
       ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME})
   endif()
 endforeach()
 
+# iOS and iOS simulator test suites
+# These are not added into "check-all", in order to run these tests, you have to
+# manually call (from the build directory).  They also require that an extra env
+# variable to select which iOS device or simulator to use, e.g.:
+# $ SANITIZER_IOSSIM_TEST_DEVICE_IDENTIFIER=BBE44C1C-8AAA-4000-8D06-91C89ED58172
+# $ ./bin/llvm-lit ./projects/compiler-rt/test/asan/IOSSimI386Config
+if(APPLE)
+  set(ASAN_TEST_TARGET_CC ${COMPILER_RT_TEST_COMPILER})
+  set(ASAN_TEST_IOS "1")
+  pythonize_bool(ASAN_TEST_IOS)
+  set(ASAN_TEST_DYNAMIC True)
+
+  foreach(arch ${DARWIN_iossim_ARCHS})
+    set(ASAN_TEST_IOSSIM "1")
+    pythonize_bool(ASAN_TEST_IOSSIM)
+    set(ASAN_TEST_TARGET_ARCH ${arch})
+    set(ASAN_TEST_TARGET_CFLAGS "-arch ${arch} -isysroot ${DARWIN_iossim_SYSROOT} ${COMPILER_RT_TEST_COMPILER_CFLAGS}")
+    set(ASAN_TEST_CONFIG_SUFFIX "-${arch}-iossim")
+    get_bits_for_arch(${arch} ASAN_TEST_BITS)
+    string(TOUPPER ${arch} ARCH_UPPER_CASE)
+    set(CONFIG_NAME "IOSSim${ARCH_UPPER_CASE}Config")
+    configure_lit_site_cfg(
+      ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.in
+      ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg
+      )
+  endforeach()
+
+  foreach (arch ${DARWIN_ios_ARCHS})
+    set(ASAN_TEST_IOSSIM "0")
+    pythonize_bool(ASAN_TEST_IOSSIM)
+    set(ASAN_TEST_TARGET_ARCH ${arch})
+    set(ASAN_TEST_TARGET_CFLAGS "-arch ${arch} -isysroot ${DARWIN_ios_SYSROOT} ${COMPILER_RT_TEST_COMPILER_CFLAGS}")
+    set(ASAN_TEST_CONFIG_SUFFIX "-${arch}-ios")
+    get_bits_for_arch(${arch} ASAN_TEST_BITS)
+    string(TOUPPER ${arch} ARCH_UPPER_CASE)
+    set(CONFIG_NAME "IOS${ARCH_UPPER_CASE}Config")
+    configure_lit_site_cfg(
+      ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.in
+      ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg
+      )
+  endforeach()
+endif()
+
 # Add unit tests.
 if(COMPILER_RT_INCLUDE_TESTS)
   set(ASAN_TEST_DYNAMIC False)
   configure_lit_site_cfg(
     ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.in
     ${CMAKE_CURRENT_BINARY_DIR}/Unit/lit.site.cfg)
   if(COMPILER_RT_ASAN_HAS_STATIC_RUNTIME)
     set(ASAN_TEST_DYNAMIC True)

compiler-rt/trunk/test/asan/lit.site.cfg.in

 @LIT_SITE_CFG_IN_HEADER@
 
 # Tool-specific config options.
 config.name_suffix = "@ASAN_TEST_CONFIG_SUFFIX@"
 config.asan_lit_source_dir = "@ASAN_LIT_SOURCE_DIR@"
 config.target_cflags = "@ASAN_TEST_TARGET_CFLAGS@"
 config.clang = "@ASAN_TEST_TARGET_CC@"
 config.bits = "@ASAN_TEST_BITS@"
 config.android = "@ANDROID@"
+config.ios = @ASAN_TEST_IOS_PYBOOL@
+config.iossim = @ASAN_TEST_IOSSIM_PYBOOL@
 config.asan_dynamic = @ASAN_TEST_DYNAMIC@
 config.target_arch = "@ASAN_TEST_TARGET_ARCH@"
 
 # Load common config for all compiler-rt lit tests.
 lit_config.load_config(config, "@COMPILER_RT_BINARY_DIR@/test/lit.common.configured")
 
 # Load tool-specific config that would do the real work.
 lit_config.load_config(config, "@ASAN_LIT_SOURCE_DIR@/lit.cfg")

compiler-rt/trunk/test/lit.common.cfg

 
 # Use ugly construction to explicitly prohibit "clang", "clang++" etc.
 # in RUN lines.
 config.substitutions.append(
     (' clang', """\n\n*** Do not use 'clangXXX' in tests,
      instead define '%clangXXX' substitution in lit config. ***\n\n""") )
 
 # Allow tests to be executed on a simulator or remotely.
+if config.emulator:
   config.substitutions.append( ('%run', config.emulator) )
+elif config.ios:
+  device_id_env = "SANITIZER_IOSSIM_TEST_DEVICE_IDENTIFIER" if config.iossim else "SANITIZER_IOS_TEST_DEVICE_IDENTIFIER"
+  if device_id_env in os.environ: config.environment[device_id_env] = os.environ[device_id_env]
+  ios_commands_dir = os.path.join(config.compiler_rt_src_root, "test", "sanitizer_common", "ios_commands")
+  run_wrapper = os.path.join(ios_commands_dir, "iossim_run.py" if config.iossim else "ios_run.py")
+  config.substitutions.append(('%run', run_wrapper))
+else:
+  config.substitutions.append( ('%run', "") )
 
 # Define CHECK-%os to check for OS-dependent output.
 config.substitutions.append( ('CHECK-%os', ("CHECK-" + config.host_os)))
 
 # Define %arch to check for architecture-dependent output.
 config.substitutions.append( ('%arch', (config.host_arch)))
 
 if config.host_os == 'Windows':

compiler-rt/trunk/test/lit.common.configured.in

 set_default("llvm_shlib_dir", "@LLVM_LIBRARY_OUTPUT_INTDIR@")
 set_default("gold_executable", "@GOLD_EXECUTABLE@")
 set_default("clang", "@COMPILER_RT_TEST_COMPILER@")
 set_default("compiler_id", "@COMPILER_RT_TEST_COMPILER_ID@")
 set_default("python_executable", "@PYTHON_EXECUTABLE@")
 set_default("compiler_rt_debug", @COMPILER_RT_DEBUG_PYBOOL@)
 set_default("compiler_rt_libdir", "@COMPILER_RT_LIBRARY_OUTPUT_DIR@")
 set_default("emulator", "@COMPILER_RT_EMULATOR@")
+set_default("ios", False)
+set_default("iossim", False)
 set_default("sanitizer_can_use_cxxabi", @SANITIZER_CAN_USE_CXXABI_PYBOOL@)
 set_default("has_lld", @COMPILER_RT_HAS_LLD_PYBOOL@)
 set_default("can_symbolize", @CAN_SYMBOLIZE@)
 set_default("use_lld", False)
 set_default("use_thinlto", False)
 config.available_features.add('target-is-%s' % config.target_arch)
 
 # LLVM tools dir can be passed in lit parameters, so try to

compiler-rt/trunk/test/sanitizer_common/ios_commands/iossim_env.py

+#!/usr/bin/python
+
+import os, sys, subprocess
+
+
+idx = 1
+for arg in sys.argv[1:]:
+  if not "=" in arg:
+    break
+  idx += 1
+  (argname, argval) = arg.split("=")
+  os.environ["SIMCTL_CHILD_" + argname] = argval
+
+exitcode = subprocess.call(sys.argv[idx:])
+if exitcode > 125:
+  exitcode = 126
+sys.exit(exitcode)

compiler-rt/trunk/test/sanitizer_common/ios_commands/iossim_run.py

+#!/usr/bin/python
+
+import os, sys, subprocess
+
+
+if not "SANITIZER_IOSSIM_TEST_DEVICE_IDENTIFIER" in os.environ:
+  raise EnvironmentError("Specify SANITIZER_IOSSIM_TEST_DEVICE_IDENTIFIER to select which simulator to use.")
+
+device_id = os.environ["SANITIZER_IOSSIM_TEST_DEVICE_IDENTIFIER"]
+
+if "ASAN_OPTIONS" in os.environ:
+    os.environ["SIMCTL_CHILD_ASAN_OPTIONS"] = os.environ["ASAN_OPTIONS"]
+
+exitcode = subprocess.call(["xcrun", "simctl", "spawn", device_id] + sys.argv[1:])
+if exitcode > 125:
+  exitcode = 126
+sys.exit(exitcode)