[scudo] Scudo thread specific data refactor, part 3

[cryptoad]

Summary

Previous parts: D38139, D38183.

In this part of the refactor, we abstract the Linux vs Android TSD dissociation
in favor of a Exclusive vs Shared one, allowing for easier platform introduction
and configuration.

Most of this change consist of shuffling the files around to reflect the new
organization.

We introduce scudo_platform.h where platform specific definition lie. This
involves the TSD model and the platform specific allocator parameters. In an
upcoming CL, those will be configurable via defines, but we currently stick
with conservative defaults.

[alekseyshl]

lib/scudo/scudo_platform.h

20

... on this platform.

31

Maybe

#if SANITIZER_ANDROID || SANITIZER_FUCHSIA
  ...
#elif SANITIZER_LINUX && !SANITIZER_ANDROID
  ...
#else
  error
#endif

would be a bit more robust?

lib/scudo/scudo_tsd.h

58

const bool UnlockRequired;

65

Fix the comment, they are not platform specific anymore.

[cryptoad]

Addressing Aleksey's comments.

[cryptoad]

Unsubmitted inline response.

lib/scudo/scudo_tsd.h

58

I am not using a constructor but an init() function.
So const can't be used as UnlockRequired gets assigned.

lib/scudo/CMakeLists.txt

 add_compiler_rt_component(scudo)
 
 include_directories(..)
 
 set(SCUDO_CFLAGS ${SANITIZER_COMMON_CFLAGS})
 # SANITIZER_COMMON_CFLAGS include -fno-builtin, but we actually want builtins!
 list(APPEND SCUDO_CFLAGS -fbuiltin)
 append_rtti_flag(OFF SCUDO_CFLAGS)
 
 set(SCUDO_SOURCES
   scudo_allocator.cpp
   scudo_flags.cpp
   scudo_crc32.cpp
   scudo_interceptors.cpp
   scudo_new_delete.cpp
   scudo_termination.cpp
-  scudo_tls_android.cpp
-  scudo_tls_linux.cpp
+  scudo_tsd_exclusive.cpp
+  scudo_tsd_shared.cpp
   scudo_utils.cpp)
 
 # Enable the SSE 4.2 instruction set for scudo_crc32.cpp, if available.
 if (COMPILER_RT_HAS_MSSE4_2_FLAG)
   set_source_files_properties(scudo_crc32.cpp PROPERTIES COMPILE_FLAGS -msse4.2)
 endif()
 
 # Enable the AArch64 CRC32 feature for scudo_crc32.cpp, if available.

lib/scudo/scudo_allocator.h

 //===-- scudo_allocator.h ---------------------------------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 ///
 /// Header for scudo_allocator.cpp.
 ///
 //===----------------------------------------------------------------------===//
 
 #ifndef SCUDO_ALLOCATOR_H_
 #define SCUDO_ALLOCATOR_H_
 
-#include "sanitizer_common/sanitizer_allocator.h"
-
-#if !SANITIZER_LINUX
-# error "The Scudo hardened allocator is currently only supported on Linux."
-#endif
+#include "scudo_platform.h"
 
 namespace __scudo {
 
 enum AllocType : u8 {
   FromMalloc    = 0,  // Memory block came from malloc, realloc, calloc, etc.
   FromNew       = 1,  // Memory block came from operator new.
   FromNewArray  = 2,  // Memory block came from operator new [].
   FromMemalign  = 3,  // Memory block came from memalign, posix_memalign, etc.

 const uptr MaxAlignment = 1 << MaxAlignmentLog;
 
 const uptr ChunkHeaderSize = sizeof(PackedHeader);
 const uptr AlignedChunkHeaderSize =
     (ChunkHeaderSize + MinAlignment - 1) & ~(MinAlignment - 1);
 
 #if SANITIZER_CAN_USE_ALLOCATOR64
 const uptr AllocatorSpace = ~0ULL;
-# if defined(__aarch64__) && SANITIZER_ANDROID
-const uptr AllocatorSize = 0x4000000000ULL;  // 256G.
-# elif defined(__aarch64__)
-const uptr AllocatorSize = 0x10000000000ULL;  // 1T.
-# else
-const uptr AllocatorSize = 0x40000000000ULL;  // 4T.
-# endif
-typedef DefaultSizeClassMap SizeClassMap;
 struct AP64 {
   static const uptr kSpaceBeg = AllocatorSpace;
   static const uptr kSpaceSize = AllocatorSize;
   static const uptr kMetadataSize = 0;
   typedef __scudo::SizeClassMap SizeClassMap;
   typedef NoOpMapUnmapCallback MapUnmapCallback;
   static const uptr kFlags =
       SizeClassAllocator64FlagMasks::kRandomShuffleChunks;
 };
 typedef SizeClassAllocator64<AP64> PrimaryAllocator;
 #else
 // Currently, the 32-bit Sanitizer allocator has not yet benefited from all the
 // security improvements brought to the 64-bit one. This makes the 32-bit
 // version of Scudo slightly less toughened.
-static const uptr RegionSizeLog = 20;
 static const uptr NumRegions = SANITIZER_MMAP_RANGE_SIZE >> RegionSizeLog;
 # if SANITIZER_WORDSIZE == 32
 typedef FlatByteMap<NumRegions> ByteMap;
 # elif SANITIZER_WORDSIZE == 64
 typedef TwoLevelByteMap<(NumRegions >> 12), 1 << 12> ByteMap;
 # endif  // SANITIZER_WORDSIZE
-typedef DefaultSizeClassMap SizeClassMap;
 struct AP32 {
   static const uptr kSpaceBeg = 0;
   static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE;
   static const uptr kMetadataSize = 0;
   typedef __scudo::SizeClassMap SizeClassMap;
   static const uptr kRegionSizeLog = RegionSizeLog;
   typedef __scudo::ByteMap ByteMap;
   typedef NoOpMapUnmapCallback MapUnmapCallback;

lib/scudo/scudo_allocator.cpp

 /// heap corruption vulnerabilities. It provides a checksum-guarded chunk
 /// header, a delayed free list, and additional sanity checks.
 ///
 //===----------------------------------------------------------------------===//
 
 #include "scudo_allocator.h"
 #include "scudo_crc32.h"
 #include "scudo_flags.h"
-#include "scudo_tls.h"
+#include "scudo_tsd.h"
 #include "scudo_utils.h"
 
 #include "sanitizer_common/sanitizer_allocator_checks.h"
 #include "sanitizer_common/sanitizer_allocator_interface.h"
 #include "sanitizer_common/sanitizer_errno.h"
 #include "sanitizer_common/sanitizer_quarantine.h"
 
 #include <string.h>

lib/scudo/scudo_platform.h

+//===-- scudo_platform.h ----------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// Scudo platform specific definitions.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_PLATFORM_H_
+#define SCUDO_PLATFORM_H_
+
+#include "sanitizer_common/sanitizer_allocator.h"
+
+#if !SANITIZER_LINUX && !SANITIZER_FUCHSIA
+# error "The Scudo hardened allocator is currently not supported this platform."
+#endif
+
+// Android and Fuchsia use a pool of TSDs shared between threads.
+#if SANITIZER_ANDROID || SANITIZER_FUCHSIA
+# define SCUDO_TSD_EXCLUSIVE 0
+#endif  // SANITIZER_FUCHSIA || SANITIZER_FUCHSIA
+
+// Non-Android Linux use an exclusive TSD per thread.
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+# define SCUDO_TSD_EXCLUSIVE 1
+#endif  // SANITIZER_LINUX && !SANITIZER_ANDROID
+
+namespace __scudo {
+
+#if SANITIZER_CAN_USE_ALLOCATOR64
+# if defined(__aarch64__) && SANITIZER_ANDROID
+const uptr AllocatorSize = 0x2000000000ULL;  // 128G.
+typedef VeryCompactSizeClassMap SizeClassMap;
+# elif defined(__aarch64__)
+const uptr AllocatorSize = 0x10000000000ULL;  // 1T.
+typedef CompactSizeClassMap SizeClassMap;
+# else
+const uptr AllocatorSize = 0x40000000000ULL;  // 4T.
+typedef CompactSizeClassMap SizeClassMap;
+# endif
+#else
+# if SANITIZER_ANDROID
+static const uptr RegionSizeLog = 19;
+typedef VeryCompactSizeClassMap SizeClassMap;
+# else
+static const uptr RegionSizeLog = 20;
+typedef CompactSizeClassMap SizeClassMap;
+# endif
+#endif  // SANITIZER_CAN_USE_ALLOCATOR64
+
+}  // namespace __scudo
+
+#endif // SCUDO_PLATFORM_H_

lib/scudo/scudo_tls.h

-//===-- scudo_tls.h ---------------------------------------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-///
-/// Scudo thread local structure definition.
-/// Implementation will differ based on the thread local storage primitives
-/// offered by the underlying platform.
-///
-//===----------------------------------------------------------------------===//
-
-#ifndef SCUDO_TLS_H_
-#define SCUDO_TLS_H_
-
-#include "scudo_allocator.h"
-#include "scudo_utils.h"
-
-#include "sanitizer_common/sanitizer_linux.h"
-#include "sanitizer_common/sanitizer_platform.h"
-
-namespace __scudo {
-
-struct ALIGNED(64) ScudoTSD {
-  AllocatorCache Cache;
-  ScudoPrng Prng;
-  uptr QuarantineCachePlaceHolder[4];
-
-  void init(bool Shared);
-  void commitBack();
-
-  INLINE bool tryLock() {
-    if (Mutex.TryLock()) {
-      atomic_store_relaxed(&Precedence, 0);
-      return true;
-    }
-    if (atomic_load_relaxed(&Precedence) == 0)
-      atomic_store_relaxed(&Precedence, NanoTime());
-    return false;
-  }
-
-  INLINE void lock() {
-    Mutex.Lock();
-    atomic_store_relaxed(&Precedence, 0);
-  }
-
-  INLINE void unlock() {
-    if (!UnlockRequired)
-      return;
-    Mutex.Unlock();
-  }
-
-  INLINE u64 getPrecedence() {
-    return atomic_load_relaxed(&Precedence);
-  }
-
- private:
-  bool UnlockRequired;
-  StaticSpinMutex Mutex;
-  atomic_uint64_t Precedence;
-};
-
-void initThread(bool MinimalInit);
-
-// Platform specific fastpath functions definitions.
-#include "scudo_tls_android.inc"
-#include "scudo_tls_linux.inc"
-
-}  // namespace __scudo
-
-#endif  // SCUDO_TLS_H_

lib/scudo/scudo_tsd.h

-//===-- scudo_tls.h ---------------------------------------------*- C++ -*-===//
+//===-- scudo_tsd.h ---------------------------------------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 ///
-/// Scudo thread local structure definition.
+/// Scudo thread specific data definition.
 /// Implementation will differ based on the thread local storage primitives
 /// offered by the underlying platform.
 ///
 //===----------------------------------------------------------------------===//
 
-#ifndef SCUDO_TLS_H_
-#define SCUDO_TLS_H_
+#ifndef SCUDO_TSD_H_
+#define SCUDO_TSD_H_
 
 #include "scudo_allocator.h"
 #include "scudo_utils.h"
 
-#include "sanitizer_common/sanitizer_linux.h"
-#include "sanitizer_common/sanitizer_platform.h"
-
 namespace __scudo {
 
 struct ALIGNED(64) ScudoTSD {
   AllocatorCache Cache;
   ScudoPrng Prng;
   uptr QuarantineCachePlaceHolder[4];
 
   void init(bool Shared);
INLINE void unlock() {
     Mutex.Unlock();
   }
 
   INLINE u64 getPrecedence() {
     return atomic_load_relaxed(&Precedence);
   }
 
  private:
   bool UnlockRequired;
   StaticSpinMutex Mutex;
   atomic_uint64_t Precedence;
 };
 
 void initThread(bool MinimalInit);
 
 // Platform specific fastpath functions definitions.
-#include "scudo_tls_android.inc"
-#include "scudo_tls_linux.inc"
+#include "scudo_tsd_exclusive.inc"
+#include "scudo_tsd_shared.inc"
 
 }  // namespace __scudo
 
-#endif  // SCUDO_TLS_H_
+#endif  // SCUDO_TSD_H_

lib/scudo/scudo_tsd_exclusive.cpp

-//===-- scudo_tls_linux.cpp -------------------------------------*- C++ -*-===//
+//===-- scudo_tsd_exclusive.cpp ---------------------------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 ///
-/// Scudo thread local structure implementation for platforms supporting
-/// thread_local.
+/// Scudo exclusive TSD implementation.
 ///
 //===----------------------------------------------------------------------===//
 
-#include "sanitizer_common/sanitizer_platform.h"
+#include "scudo_tsd.h"
 
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
-
-#include "scudo_tls.h"
+#if SCUDO_TSD_EXCLUSIVE
 
 #include <pthread.h>
 
 namespace __scudo {
 
 static pthread_once_t GlobalInitialized = PTHREAD_ONCE_INIT;
 static pthread_key_t PThreadKey;
 
void initThread(bool MinimalInit) {
   CHECK_EQ(pthread_setspecific(PThreadKey, reinterpret_cast<void *>(
       GetPthreadDestructorIterations())), 0);
   TSD.init(/*Shared=*/false);
   ScudoThreadState = ThreadInitialized;
 }
 
 }  // namespace __scudo
 
-#endif  // SANITIZER_LINUX && !SANITIZER_ANDROID
+#endif  // SCUDO_TSD_EXCLUSIVE

lib/scudo/scudo_tsd_exclusive.inc

-//===-- scudo_tls_linux.inc -------------------------------------*- C++ -*-===//
+//===-- scudo_tsd_exclusive.inc ---------------------------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 ///
-/// Scudo thread local structure fastpath functions implementation for platforms
-/// supporting thread_local.
+/// Scudo exclusive TSD fastpath functions implementation.
 ///
 //===----------------------------------------------------------------------===//
 
-#ifndef SCUDO_TLS_H_
-# error "This file must be included inside scudo_tls.h."
-#endif  // SCUDO_TLS_H_
+#ifndef SCUDO_TSD_H_
+# error "This file must be included inside scudo_tsd.h."
+#endif  // SCUDO_TSD_H_
 
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
+#if SCUDO_TSD_EXCLUSIVE
 
 enum ThreadState : u8 {
   ThreadNotInitialized = 0,
   ThreadInitialized,
   ThreadTornDown,
 };
 __attribute__((tls_model("initial-exec")))
 extern THREADLOCAL ThreadState ScudoThreadState;

 ALWAYS_INLINE ScudoTSD *getTSDAndLock() {
   if (UNLIKELY(ScudoThreadState != ThreadInitialized)) {
     FallbackTSD.lock();
     return &FallbackTSD;
   }
   return &TSD;
 }
 
-#endif  // SANITIZER_LINUX && !SANITIZER_ANDROID
+#endif  // SCUDO_TSD_EXCLUSIVE

lib/scudo/scudo_tsd_shared.cpp

-//===-- scudo_tls_android.cpp -----------------------------------*- C++ -*-===//
+//===-- scudo_tsd_shared.cpp ------------------------------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 ///
-/// Scudo thread local structure implementation for Android.
+/// Scudo shared TSD implementation.
 ///
 //===----------------------------------------------------------------------===//
 
-#include "sanitizer_common/sanitizer_platform.h"
+#include "scudo_tsd.h"
 
-#if SANITIZER_LINUX && SANITIZER_ANDROID
-
-#include "scudo_tls.h"
+#if !SCUDO_TSD_EXCLUSIVE
 
 #include <pthread.h>
 
 namespace __scudo {
 
 static pthread_once_t GlobalInitialized = PTHREAD_ONCE_INIT;
 static pthread_key_t PThreadKey;
 
ScudoTSD *getTSDAndLockSlow() {
   // Last resort, stick with the current one.
   TSD = reinterpret_cast<ScudoTSD *>(*get_android_tls_ptr());
   TSD->lock();
   return TSD;
 }
 
 }  // namespace __scudo
 
-#endif  // SANITIZER_LINUX && SANITIZER_ANDROID
+#endif  // !SCUDO_TSD_EXCLUSIVE

lib/scudo/scudo_tsd_shared.inc

-//===-- scudo_tls_android.inc -----------------------------------*- C++ -*-===//
+//===-- scudo_tsd_shared.inc ------------------------------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 ///
-/// Scudo thread local structure fastpath functions implementation for Android.
+/// Scudo shared TSD fastpath functions implementation.
 ///
 //===----------------------------------------------------------------------===//
 
-#ifndef SCUDO_TLS_H_
-# error "This file must be included inside scudo_tls.h."
-#endif  // SCUDO_TLS_H_
+#ifndef SCUDO_TSD_H_
+# error "This file must be included inside scudo_tsd.h."
+#endif  // SCUDO_TSD_H_
 
-#if SANITIZER_LINUX && SANITIZER_ANDROID
+#if !SCUDO_TSD_EXCLUSIVE
 
 ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) {
   if (LIKELY(*get_android_tls_ptr()))
     return;
   initThread(MinimalInit);
 }
 
 ScudoTSD *getTSDAndLockSlow();
 
 ALWAYS_INLINE ScudoTSD *getTSDAndLock() {
   ScudoTSD *TSD = reinterpret_cast<ScudoTSD *>(*get_android_tls_ptr());
   CHECK(TSD && "No TSD associated with the current thread!");
   // Try to lock the currently associated context.
   if (TSD->tryLock())
     return TSD;
   // If it failed, go the slow path.
   return getTSDAndLockSlow();
 }
 
-#endif  // SANITIZER_LINUX && SANITIZER_ANDROID
+#endif  // !SCUDO_TSD_EXCLUSIVE