diff --git a/compiler-rt/lib/scudo/standalone/combined.h b/compiler-rt/lib/scudo/standalone/combined.h --- a/compiler-rt/lib/scudo/standalone/combined.h +++ b/compiler-rt/lib/scudo/standalone/combined.h @@ -31,15 +31,23 @@ static gwp_asan::GuardedPoolAllocator GuardedAlloc; #endif // GWP_ASAN_HOOKS +extern "C" inline void EmptyCallback() {} + namespace scudo { -template class Allocator { +template +class Allocator { public: using PrimaryT = typename Params::Primary; using CacheT = typename PrimaryT::CacheT; - typedef Allocator ThisT; + typedef Allocator ThisT; typedef typename Params::template TSDRegistryT TSDRegistryT; + void callPostInitCallback() { + static pthread_once_t OnceControl = PTHREAD_ONCE_INIT; + pthread_once(&OnceControl, PostInitCallback); + } + struct QuarantineCallback { explicit QuarantineCallback(ThisT &Instance, CacheT &LocalCache) : Allocator(Instance), Cache(LocalCache) {} @@ -420,12 +428,14 @@ void disable() { initThreadMaybe(); TSDRegistry.disable(); + Primary.disable(); Secondary.disable(); } void enable() { initThreadMaybe(); Secondary.enable(); + Primary.enable(); TSDRegistry.enable(); } diff --git a/compiler-rt/lib/scudo/standalone/tests/tsd_test.cpp b/compiler-rt/lib/scudo/standalone/tests/tsd_test.cpp --- a/compiler-rt/lib/scudo/standalone/tests/tsd_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/tsd_test.cpp @@ -36,6 +36,7 @@ void initCache(CacheT *Cache) { memset(Cache, 0, sizeof(*Cache)); } void commitBack(scudo::TSD *TSD) {} TSDRegistryT *getTSDRegistry() { return &TSDRegistry; } + void callPostInitCallback() {} bool isInitialized() { return Initialized; } diff --git a/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp b/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp --- a/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp @@ -310,4 +310,32 @@ fclose(F); EXPECT_EQ(strncmp(Buffer, " #include #include #include @@ -113,3 +114,56 @@ for (auto &T : Threads) T.join(); } + +#if !SCUDO_FUCHSIA +TEST(ScudoWrappersCppTest, AllocAfterFork) { + std::atomic_bool Stop; + + // Create threads that simply allocate and free different sizes. + std::vector Threads; + for (size_t N = 0; N < 5; N++) { + std::thread *T = new std::thread([&Stop] { + while (!Stop) { + for (size_t SizeLog = 3; SizeLog <= 21; SizeLog++) { + char *P = new char[1UL << SizeLog]; + EXPECT_NE(P, nullptr); + // Make sure this value is not optimized away. + asm volatile("" : : "r,m"(P) : "memory"); + delete[] P; + } + } + }); + Threads.push_back(T); + } + + // Create a thread to fork and allocate. + for (size_t N = 0; N < 100; N++) { + printf("Running iteration %zu\n", N); + pid_t Pid; + if ((Pid = fork()) == 0) { + for (size_t SizeLog = 3; SizeLog <= 21; SizeLog++) { + char *P = new char[1UL << SizeLog]; + EXPECT_NE(P, nullptr); + // Make sure this value is not optimized away. + asm volatile("" : : "r,m"(P) : "memory"); + // Make sure we can touch all of the allocation. + memset(P, 0x32, 1U << SizeLog); + // EXPECT_LE(1U << SizeLog, malloc_usable_size(ptr)); + delete[] P; + } + _exit(10); + } + EXPECT_NE(-1, Pid); + int Status; + EXPECT_EQ(Pid, waitpid(Pid, &Status, 0)); + EXPECT_FALSE(WIFSIGNALED(Status)); + EXPECT_EQ(10, WEXITSTATUS(Status)); + } + + printf("Waiting for threads to complete\n"); + Stop = true; + for (auto Thread : Threads) + Thread->join(); + Threads.clear(); +} +#endif \ No newline at end of file diff --git a/compiler-rt/lib/scudo/standalone/tsd.h b/compiler-rt/lib/scudo/standalone/tsd.h --- a/compiler-rt/lib/scudo/standalone/tsd.h +++ b/compiler-rt/lib/scudo/standalone/tsd.h @@ -14,6 +14,7 @@ #include "mutex.h" #include // for PTHREAD_DESTRUCTOR_ITERATIONS +#include // With some build setups, this might still not be defined. #ifndef PTHREAD_DESTRUCTOR_ITERATIONS diff --git a/compiler-rt/lib/scudo/standalone/tsd_exclusive.h b/compiler-rt/lib/scudo/standalone/tsd_exclusive.h --- a/compiler-rt/lib/scudo/standalone/tsd_exclusive.h +++ b/compiler-rt/lib/scudo/standalone/tsd_exclusive.h @@ -11,8 +11,6 @@ #include "tsd.h" -#include - namespace scudo { enum class ThreadState : u8 { @@ -90,6 +88,7 @@ pthread_setspecific(PThreadKey, reinterpret_cast(Instance)), 0); ThreadTSD.initLinkerInitialized(Instance); State = ThreadState::Initialized; + Instance->callPostInitCallback(); } pthread_key_t PThreadKey; diff --git a/compiler-rt/lib/scudo/standalone/tsd_shared.h b/compiler-rt/lib/scudo/standalone/tsd_shared.h --- a/compiler-rt/lib/scudo/standalone/tsd_shared.h +++ b/compiler-rt/lib/scudo/standalone/tsd_shared.h @@ -12,8 +12,6 @@ #include "linux.h" // for getAndroidTlsPtr() #include "tsd.h" -#include - namespace scudo { template struct TSDRegistrySharedT { @@ -117,6 +115,7 @@ // Initial context assignment is done in a plain round-robin fashion. const u32 Index = atomic_fetch_add(&CurrentIndex, 1U, memory_order_relaxed); setCurrentTSD(&TSDs[Index % NumberOfTSDs]); + Instance->callPostInitCallback(); } NOINLINE TSD *getTSDAndLockSlow(TSD *CurrentTSD) { diff --git a/compiler-rt/lib/scudo/standalone/wrappers_c.cpp b/compiler-rt/lib/scudo/standalone/wrappers_c.cpp --- a/compiler-rt/lib/scudo/standalone/wrappers_c.cpp +++ b/compiler-rt/lib/scudo/standalone/wrappers_c.cpp @@ -18,22 +18,23 @@ #include #include -static scudo::Allocator Allocator; +#define SCUDO_PREFIX(name) name +#define SCUDO_ALLOCATOR Allocator + +extern "C" INTERFACE void SCUDO_PREFIX(malloc_postinit)(); +static scudo::Allocator + SCUDO_ALLOCATOR; // Pointer to the static allocator so that the C++ wrappers can access it. // Technically we could have a completely separated heap for C & C++ but in // reality the amount of cross pollination between the two is staggering. -scudo::Allocator *AllocatorPtr = &Allocator; - -extern "C" { +scudo::Allocator * + CONCATENATE(SCUDO_ALLOCATOR, Ptr) = &SCUDO_ALLOCATOR; -#define SCUDO_PREFIX(name) name -#define SCUDO_ALLOCATOR Allocator #include "wrappers_c.inc" + #undef SCUDO_ALLOCATOR #undef SCUDO_PREFIX -INTERFACE void __scudo_print_stats(void) { Allocator.printStats(); } - -} // extern "C" +extern "C" INTERFACE void __scudo_print_stats(void) { Allocator.printStats(); } #endif // !SCUDO_ANDROID || !_BIONIC diff --git a/compiler-rt/lib/scudo/standalone/wrappers_c.inc b/compiler-rt/lib/scudo/standalone/wrappers_c.inc --- a/compiler-rt/lib/scudo/standalone/wrappers_c.inc +++ b/compiler-rt/lib/scudo/standalone/wrappers_c.inc @@ -17,6 +17,8 @@ #define SCUDO_MALLOC_ALIGNMENT FIRST_32_SECOND_64(8U, 16U) #endif +extern "C" { + INTERFACE WEAK void *SCUDO_PREFIX(calloc)(size_t nmemb, size_t size) { scudo::uptr Product; if (UNLIKELY(scudo::checkForCallocOverflow(size, nmemb, &Product))) { @@ -141,11 +143,31 @@ return 0; } +static pthread_mutex_t SCUDO_PREFIX(Mutex) = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t SCUDO_PREFIX(Conditional) = PTHREAD_COND_INITIALIZER; +static bool SCUDO_PREFIX(Disabled) = false; + +INTERFACE WEAK void SCUDO_PREFIX(malloc_enable)() { + pthread_mutex_lock(&SCUDO_PREFIX(Mutex)); + SCUDO_ALLOCATOR.enable(); + SCUDO_PREFIX(Disabled) = false; + pthread_cond_broadcast(&SCUDO_PREFIX(Conditional)); + pthread_mutex_unlock(&SCUDO_PREFIX(Mutex)); +} + INTERFACE WEAK void SCUDO_PREFIX(malloc_disable)() { + pthread_mutex_lock(&SCUDO_PREFIX(Mutex)); + while (SCUDO_PREFIX(Disabled)) + pthread_cond_wait(&SCUDO_PREFIX(Conditional), &SCUDO_PREFIX(Mutex)); SCUDO_ALLOCATOR.disable(); + SCUDO_PREFIX(Disabled) = true; + pthread_mutex_unlock(&SCUDO_PREFIX(Mutex)); } -INTERFACE WEAK void SCUDO_PREFIX(malloc_enable)() { SCUDO_ALLOCATOR.enable(); } +void SCUDO_PREFIX(malloc_postinit)() { + pthread_atfork(SCUDO_PREFIX(malloc_disable), SCUDO_PREFIX(malloc_enable), + SCUDO_PREFIX(malloc_enable)); +} INTERFACE WEAK int SCUDO_PREFIX(mallopt)(int param, UNUSED int value) { if (param == M_DECAY_TIME) { @@ -176,3 +198,5 @@ fputs("", stream); return 0; } + +} // extern "C" diff --git a/compiler-rt/lib/scudo/standalone/wrappers_c_bionic.cpp b/compiler-rt/lib/scudo/standalone/wrappers_c_bionic.cpp --- a/compiler-rt/lib/scudo/standalone/wrappers_c_bionic.cpp +++ b/compiler-rt/lib/scudo/standalone/wrappers_c_bionic.cpp @@ -18,22 +18,40 @@ #include #include -static scudo::Allocator Allocator; -static scudo::Allocator SvelteAllocator; - -extern "C" { - // Regular MallocDispatch definitions. #define SCUDO_PREFIX(name) CONCATENATE(scudo_, name) #define SCUDO_ALLOCATOR Allocator + +extern "C" INTERFACE void SCUDO_PREFIX(malloc_postinit)(); +static scudo::Allocator + SCUDO_ALLOCATOR; +// Pointer to the static allocator so that the C++ wrappers can access it. +// Technically we could have a completely separated heap for C & C++ but in +// reality the amount of cross pollination between the two is staggering. +scudo::Allocator * + CONCATENATE(SCUDO_ALLOCATOR, Ptr) = &SCUDO_ALLOCATOR; + #include "wrappers_c.inc" + #undef SCUDO_ALLOCATOR #undef SCUDO_PREFIX // Svelte MallocDispatch definitions. #define SCUDO_PREFIX(name) CONCATENATE(scudo_svelte_, name) #define SCUDO_ALLOCATOR SvelteAllocator + +extern "C" INTERFACE void SCUDO_PREFIX(malloc_postinit)(); +static scudo::Allocator + SCUDO_ALLOCATOR; +// Pointer to the static allocator so that the C++ wrappers can access it. +// Technically we could have a completely separated heap for C & C++ but in +// reality the amount of cross pollination between the two is staggering. +scudo::Allocator * + CONCATENATE(SCUDO_ALLOCATOR, Ptr) = &SCUDO_ALLOCATOR; + #include "wrappers_c.inc" + #undef SCUDO_ALLOCATOR #undef SCUDO_PREFIX @@ -44,6 +62,4 @@ SvelteAllocator.printStats(); } -} // extern "C" - #endif // SCUDO_ANDROID && _BIONIC diff --git a/compiler-rt/lib/scudo/standalone/wrappers_cpp.cpp b/compiler-rt/lib/scudo/standalone/wrappers_cpp.cpp --- a/compiler-rt/lib/scudo/standalone/wrappers_cpp.cpp +++ b/compiler-rt/lib/scudo/standalone/wrappers_cpp.cpp @@ -15,7 +15,8 @@ #include -extern scudo::Allocator *AllocatorPtr; +extern "C" INTERFACE void malloc_postinit(); +extern scudo::Allocator *AllocatorPtr; namespace std { struct nothrow_t {};