[scudo] [android] Use ELF TLS on Android API >= 30.

[hctim]

Summary

On Android 11 and newer, we use Scudo standalone as the platform
allocator inside of libc. This causes a problem, as the platform version
of scudo owns the sanitizer TLS slot. When we run compiler-rt-based
scudo tests on a newer device, these two version of Scudo fight over the
slot.

This used to not break things as the compiler-rt version would just
clobber the TLS slot, and the platform version was never invoked. After
https://android-review.googlesource.com/c/platform/bionic/+/1299034/
however, the platform version re-clobbers the TLS slot after it's been
initialised by the constructors for the compiler-rt version under test.

To work around this, on newer devices that support ELF TLS, we add ELF
TLS support to compiler-rt scudo. This requires targeting a higher API
level (at least 29). If the user is targeting a lower API level and is
running an Android 11-or-greater device, the tests will be automatically
disabled as there's no reasonable way to resolve the conflict.

[hctim]

compiler-rt/CMakeLists.txt

280

FWIW - I'm fairly sure this was broken before (Android 10 was the first with ELF TLS at API level 29?), but it was just untested on the bots because they're currently running at API level 24.

[cryptoad]

Thanks Mitch! Looks reasonable to me.

compiler-rt/lib/scudo/scudo_tsd_shared.cpp

53

nit: we will fall in this without the || condition?

compiler-rt/lib/scudo/scudo_tsd_shared.inc

29

nit: we will fall in this without the || condition?

[oontvoo]

compiler-rt/lib/scudo/CMakeLists.txt

29

shouldn't this be inherited from SANITIZER_COMMON_CFLAGS?

[oontvoo]

compiler-rt/test/lit.common.cfg.py

370–376

nit: this whole block might be better in compiler-rt/test/scudo/lit.cfg.py because it's not used anywhere else outside of scudo

[hctim]

We're actually going to kill support for compiler-rt based scudo on Android, so this patch should become obsolete soon. Preferably, we'd like to delete the entire thing but it seems like it might be used for Windows builds experimentally...

Patches to port GWP-ASan tests to scudo-standalone should be coming soon.

compiler-rt/CMakeLists.txt

 append_string_if(COMPILER_RT_HAS_STD_CXX14_FLAG -std=c++14 CMAKE_CXX_FLAGS)
 
 # Emulate C99 and C++11's __func__ for MSVC prior to 2013 CTP.
 if(NOT COMPILER_RT_HAS_FUNC_SYMBOL)
   add_definitions(-D__func__=__FUNCTION__)
 endif()
 
 # Provide some common commmandline flags for Sanitizer runtimes.
-if("${ANDROID_API_LEVEL}" GREATER_EQUAL 28)
+if("${ANDROID_API_LEVEL}" GREATER_EQUAL 29)
   list(APPEND SANITIZER_COMMON_CFLAGS -fno-emulated-tls)
   string(APPEND COMPILER_RT_TEST_COMPILER_CFLAGS " -fno-emulated-tls")
+  set(ANDROID_ELF_TLS_SUPPORTED TRUE)
 endif()
 if(NOT WIN32)
   append_list_if(COMPILER_RT_HAS_FPIC_FLAG -fPIC SANITIZER_COMMON_CFLAGS)
 endif()
 append_list_if(COMPILER_RT_HAS_FNO_BUILTIN_FLAG -fno-builtin SANITIZER_COMMON_CFLAGS)
 append_list_if(COMPILER_RT_HAS_FNO_EXCEPTIONS_FLAG -fno-exceptions SANITIZER_COMMON_CFLAGS)
 if(NOT COMPILER_RT_DEBUG AND NOT APPLE)
   append_list_if(COMPILER_RT_HAS_FOMIT_FRAME_POINTER_FLAG -fomit-frame-pointer SANITIZER_COMMON_CFLAGS)
if(ANDROID)
   append_list_if(COMPILER_RT_HAS_FUSE_LD_LLD_FLAG -fuse-ld=lld SANITIZER_COMMON_LINK_FLAGS)
   append_list_if(COMPILER_RT_HAS_LLD -fuse-ld=lld COMPILER_RT_UNITTEST_LINK_FLAGS)
   if(COMPILER_RT_HAS_FUSE_LD_LLD_FLAG)
     set(COMPILER_RT_HAS_GNU_VERSION_SCRIPT_COMPAT FALSE)
   endif()
 endif()
 pythonize_bool(COMPILER_RT_HAS_LLD)
 pythonize_bool(COMPILER_RT_TEST_USE_LLD)
+pythonize_bool(ANDROID_ELF_TLS_SUPPORTED)
 
 add_subdirectory(lib)
 
 if(COMPILER_RT_INCLUDE_TESTS)
   add_subdirectory(unittests)
   add_subdirectory(test)
   # Don't build llvm-lit for runtimes-build, it will clean up map_config.
   if (COMPILER_RT_STANDALONE_BUILD AND NOT RUNTIMES_BUILD)

compiler-rt/lib/scudo/CMakeLists.txt

 list(APPEND SCUDO_DYNAMIC_LINK_FLAGS -Wl,--gc-sections)
 
 if(ANDROID)
 # Put most Sanitizer shared libraries in the global group. For more details, see
 # android-changes-for-ndk-developers.md#changes-to-library-search-order
   if (COMPILER_RT_HAS_Z_GLOBAL)
     list(APPEND SCUDO_DYNAMIC_LINK_FLAGS -Wl,-z,global)
   endif()
+  if (ANDROID_ELF_TLS_SUPPORTED)
+    list(APPEND SCUDO_CFLAGS -DSCUDO_USE_ELF_TLS -fno-emulated-tls)
+  endif()
 endif()
 
 # The minimal Scudo runtime does not inlude the UBSan runtime.
 set(SCUDO_MINIMAL_OBJECT_LIBS
   RTSanitizerCommonNoTermination
   RTSanitizerCommonLibc
   RTInterception)
 

compiler-rt/lib/scudo/scudo_tsd_shared.cpp

 pthread_key_t PThreadKey;
 
 static atomic_uint32_t CurrentIndex;
 static ScudoTSD *TSDs;
 static u32 NumberOfTSDs;
 static u32 CoPrimes[SCUDO_SHARED_TSD_POOL_SIZE];
 static u32 NumberOfCoPrimes = 0;
 
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
+#if SANITIZER_LINUX && (!SANITIZER_ANDROID || defined(SCUDO_USE_ELF_TLS))
 __attribute__((tls_model("initial-exec")))
 THREADLOCAL ScudoTSD *CurrentTSD;
 #endif
 
 static void initOnce() {
   CHECK_EQ(pthread_key_create(&PThreadKey, NULL), 0);
   initScudo();
   NumberOfTSDs = Min(Max(1U, GetNumberOfCPUsCached()),
                      static_cast<u32>(SCUDO_SHARED_TSD_POOL_SIZE));
   TSDs = reinterpret_cast<ScudoTSD *>(
       MmapOrDie(sizeof(ScudoTSD) * NumberOfTSDs, "ScudoTSDs"));
   for (u32 I = 0; I < NumberOfTSDs; I++) {
     TSDs[I].init();
     u32 A = I + 1;
     u32 B = NumberOfTSDs;
     while (B != 0) { const u32 T = A; A = B; B = T % B; }
     if (A == 1)
       CoPrimes[NumberOfCoPrimes++] = I + 1;
   }
 }
 
 ALWAYS_INLINE void setCurrentTSD(ScudoTSD *TSD) {
-#if SANITIZER_ANDROID
+#if SANITIZER_ANDROID && !defined(SCUDO_USE_ELF_TLS)
   *get_android_tls_ptr() = reinterpret_cast<uptr>(TSD);
-#elif SANITIZER_LINUX
+#elif SANITIZER_LINUX || defined(SCUDO_USE_ELF_TLS)
   CurrentTSD = TSD;
 #else
   CHECK_EQ(pthread_setspecific(PThreadKey, reinterpret_cast<void *>(TSD)), 0);
 #endif  // SANITIZER_ANDROID
 }
 
 void initThread(bool MinimalInit) {
   pthread_once(&GlobalInitialized, initOnce);

compiler-rt/lib/scudo/scudo_tsd_shared.inc

 //===-- scudo_tsd_shared.inc ------------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
 /// Scudo shared TSD fastpath functions implementation.
 ///
 //===----------------------------------------------------------------------===//
 
 #ifndef SCUDO_TSD_H_
 # error "This file must be included inside scudo_tsd.h."
 #endif  // SCUDO_TSD_H_
 
 #if !SCUDO_TSD_EXCLUSIVE
 
 extern pthread_key_t PThreadKey;
 
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
+#if SANITIZER_LINUX && (!SANITIZER_ANDROID || defined(SCUDO_USE_ELF_TLS))
 __attribute__((tls_model("initial-exec")))
 extern THREADLOCAL ScudoTSD *CurrentTSD;
 #endif
 
 ALWAYS_INLINE ScudoTSD* getCurrentTSD() {
-#if SANITIZER_ANDROID
+#if SANITIZER_ANDROID && !defined(SCUDO_USE_ELF_TLS)
   return reinterpret_cast<ScudoTSD *>(*get_android_tls_ptr());
-#elif SANITIZER_LINUX
+#elif SANITIZER_LINUX || defined(SCUDO_USE_ELF_TLS)
   return CurrentTSD;
 #else
   return reinterpret_cast<ScudoTSD *>(pthread_getspecific(PThreadKey));
 #endif  // SANITIZER_ANDROID
 }
 
 ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) {
   if (LIKELY(getCurrentTSD()))
     return;
   initThread(MinimalInit);
 }
 
 ScudoTSD *getTSDAndLockSlow(ScudoTSD *TSD);
 
 ALWAYS_INLINE ScudoTSD *getTSDAndLock(bool *UnlockRequired) {
   ScudoTSD *TSD = getCurrentTSD();
   DCHECK(TSD && "No TSD associated with the current thread!");
   *UnlockRequired = true;
   // Try to lock the currently associated context.
   if (TSD->tryLock())
     return TSD;
   // If it failed, go the slow path.
   return getTSDAndLockSlow(TSD);
 }
 
 #endif  // !SCUDO_TSD_EXCLUSIVE

compiler-rt/test/lit.common.cfg.py

if config.android:
   config.substitutions.append( ('%device_rm', "%s -s '%s' shell 'rm ' " % (adb, env['ANDROID_SERIAL']) ) )
 
   try:
     android_api_level_str = subprocess.check_output([adb, "shell", "getprop", "ro.build.version.sdk"], env=env).rstrip()
     android_api_codename = subprocess.check_output([adb, "shell", "getprop", "ro.build.version.codename"], env=env).rstrip().decode("utf-8")
   except (subprocess.CalledProcessError, OSError):
     lit_config.fatal("Failed to read ro.build.version.sdk (using '%s' as adb)" % adb)
   try:
-    android_api_level = int(android_api_level_str)
+    device_android_api_level = int(android_api_level_str)
   except ValueError:
     lit_config.fatal("Failed to read ro.build.version.sdk (using '%s' as adb): got '%s'" % (adb, android_api_level_str))
-  android_api_level = min(android_api_level, int(config.android_api_level))
+  android_api_level = min(device_android_api_level, int(config.android_api_level))
   for required in [26, 28, 30]:
     if android_api_level >= required:
       config.available_features.add('android-%s' % required)
   # FIXME: Replace with appropriate version when availible.
   if android_api_level > 30 or (android_api_level == 30 and android_api_codename == 'S'):
     config.available_features.add('android-thread-properties-api')
 
+  if device_android_api_level >= 30 and not config.android_elf_tls_supported:
+    # Device has Scudo standalone as the default allocator, and it's occupying
+    # the sanitizer TLS slot. The API level being built for doesn't support ELF
+    # TLS, and thus Scudo (vanilla) will fight the platform Scudo standalone
+    # over the TLS slot. This causes problems, so we need to disable Scudo tests
+    # in these cases.
+    config.scudo_tls_slot_battle = True
+
   # Prepare the device.
   android_tmpdir = '/data/local/tmp/Output'
   subprocess.check_call([adb, "shell", "mkdir", "-p", android_tmpdir], env=env)
   for file in config.android_files_to_push:
     subprocess.check_call([adb, "push", file, android_tmpdir], env=env)
 else:
   config.substitutions.append( ('%device_rundir', "") )
   config.substitutions.append( ('%push_to_device', "echo ") )

compiler-rt/test/lit.common.configured.in

 set_default("use_newpm", False)
 set_default("android", @ANDROID_PYBOOL@)
 set_default("android_api_level", "@ANDROID_API_LEVEL@")
 set_default("android_serial", "@ANDROID_SERIAL_FOR_TESTING@")
 set_default("android_files_to_push", [])
 set_default("have_rpc_xdr_h", @HAVE_RPC_XDR_H@)
 set_default("gwp_asan", @COMPILER_RT_HAS_GWP_ASAN_PYBOOL@)
 set_default("expensive_checks", @LLVM_ENABLE_EXPENSIVE_CHECKS_PYBOOL@)
+set_default("android_elf_tls_supported", @ANDROID_ELF_TLS_SUPPORTED_PYBOOL@)
+set_default("scudo_tls_slot_battle", False)
 config.available_features.add('target-is-%s' % config.target_arch)
 
 if config.enable_per_target_runtime_dir:
   set_default("target_suffix", "")
 elif config.android:
   if config.target_arch == "i386":
     set_default("target_suffix", "-i686-android")
   else:

compiler-rt/test/scudo/lit.cfg.py

if default_scudo_opts:
   config.environment['SCUDO_OPTIONS'] = default_scudo_opts
   default_scudo_opts += ':'
 config.substitutions.append(('%env_scudo_opts=',
                              'env SCUDO_OPTIONS=' + default_scudo_opts))
 
 # Hardened Allocator tests are currently supported on Linux only.
 if config.host_os not in ['Linux']:
    config.unsupported = True
+
+if config.scudo_tls_slot_battle:
+  config.unsupported = True