diff --git a/compiler-rt/lib/gwp_asan/common.h b/compiler-rt/lib/gwp_asan/common.h --- a/compiler-rt/lib/gwp_asan/common.h +++ b/compiler-rt/lib/gwp_asan/common.h @@ -83,6 +83,8 @@ // crash handler. This, in conjunction with the Metadata array, forms the entire // set of information required for understanding a GWP-ASan crash. struct AllocatorState { + constexpr AllocatorState() {} + // Returns whether the provided pointer is a current sampled allocation that // is owned by this pool. GWP_ASAN_ALWAYS_INLINE bool pointerIsMine(const void *Ptr) const { diff --git a/compiler-rt/lib/scudo/scudo_allocator.cpp b/compiler-rt/lib/scudo/scudo_allocator.cpp --- a/compiler-rt/lib/scudo/scudo_allocator.cpp +++ b/compiler-rt/lib/scudo/scudo_allocator.cpp @@ -303,13 +303,6 @@ bool ForceZeroContents = false) { initThreadMaybe(); -#ifdef GWP_ASAN_HOOKS - if (UNLIKELY(GuardedAlloc.shouldSample())) { - if (void *Ptr = GuardedAlloc.allocate(Size)) - return Ptr; - } -#endif // GWP_ASAN_HOOKS - if (UNLIKELY(Alignment > MaxAlignment)) { if (AllocatorMayReturnNull()) return nullptr; @@ -318,6 +311,16 @@ if (UNLIKELY(Alignment < MinAlignment)) Alignment = MinAlignment; +#ifdef GWP_ASAN_HOOKS + if (UNLIKELY(GuardedAlloc.shouldSample())) { + if (void *Ptr = GuardedAlloc.allocate(Size, Alignment)) { + if (SCUDO_CAN_USE_HOOKS && &__sanitizer_malloc_hook) + __sanitizer_malloc_hook(Ptr, Size); + return Ptr; + } + } +#endif // GWP_ASAN_HOOKS + const uptr NeededSize = RoundUpTo(Size ? Size : 1, MinAlignment) + Chunk::getHeaderSize(); const uptr AlignedSize = (Alignment > MinAlignment) ? diff --git a/compiler-rt/lib/scudo/standalone/combined.h b/compiler-rt/lib/scudo/standalone/combined.h --- a/compiler-rt/lib/scudo/standalone/combined.h +++ b/compiler-rt/lib/scudo/standalone/combined.h @@ -290,27 +290,31 @@ bool ZeroContents = false) { initThreadMaybe(); + const Options Options = Primary.Options.load(); + if (UNLIKELY(Alignment > MaxAlignment)) { + if (Options.get(OptionBit::MayReturnNull)) + return nullptr; + reportAlignmentTooBig(Alignment, MaxAlignment); + } + if (Alignment < MinAlignment) + Alignment = MinAlignment; + #ifdef GWP_ASAN_HOOKS if (UNLIKELY(GuardedAlloc.shouldSample())) { - if (void *Ptr = GuardedAlloc.allocate(roundUpTo(Size, Alignment))) + if (void *Ptr = GuardedAlloc.allocate(Size, Alignment)) { + if (UNLIKELY(&__scudo_allocate_hook)) + __scudo_allocate_hook(Ptr, Size); + Stats.add(StatAllocated, Size); return Ptr; + } } #endif // GWP_ASAN_HOOKS - const Options Options = Primary.Options.load(); const FillContentsMode FillContents = ZeroContents ? ZeroFill : TSDRegistry.getDisableMemInit() ? NoFill : Options.getFillContentsMode(); - if (UNLIKELY(Alignment > MaxAlignment)) { - if (Options.get(OptionBit::MayReturnNull)) - return nullptr; - reportAlignmentTooBig(Alignment, MaxAlignment); - } - if (Alignment < MinAlignment) - Alignment = MinAlignment; - // If the requested size happens to be 0 (more common than you might think), // allocate MinAlignment bytes on top of the header. Then add the extra // bytes required to fulfill the alignment requirements: we allocate enough @@ -503,18 +507,20 @@ // being destroyed properly. Any other heap operation will do a full init. initThreadMaybe(/*MinimalInit=*/true); + if (UNLIKELY(&__scudo_deallocate_hook)) + __scudo_deallocate_hook(Ptr); + + if (UNLIKELY(!Ptr)) + return; + #ifdef GWP_ASAN_HOOKS if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr))) { GuardedAlloc.deallocate(Ptr); + Stats.add(StatFree, GuardedAlloc.getSize(Ptr)); return; } #endif // GWP_ASAN_HOOKS - if (UNLIKELY(&__scudo_deallocate_hook)) - __scudo_deallocate_hook(Ptr); - - if (UNLIKELY(!Ptr)) - return; if (UNLIKELY(!isAligned(reinterpret_cast(Ptr), MinAlignment))) reportMisalignedPointer(AllocatorAction::Deallocating, Ptr); @@ -571,6 +577,7 @@ if (NewPtr) memcpy(NewPtr, OldPtr, (NewSize < OldSize) ? NewSize : OldSize); GuardedAlloc.deallocate(OldPtr); + Stats.add(StatFree, OldSize); return NewPtr; } #endif // GWP_ASAN_HOOKS @@ -954,7 +961,17 @@ TSDRegistryT TSDRegistry; #ifdef GWP_ASAN_HOOKS - gwp_asan::GuardedPoolAllocator GuardedAlloc; + // GuardedAlloc needs to be defined as 'static' in order to ensure that the + // class is statically initialized. initGwpAsan() is called lazily by + // malloc(), generally by the loader somewhere in libdl. If we also have + // dynamic initializers (which are emitted by clang at -O0 and gcc at all opt + // levels), then these are called /after/ the dynamic loader, in libc init. + // This causes a double-initialization of GWP-ASan, and no allocations after + // libc's init will go to GWP-ASan (as it was value-initialized dynamically), + // and any allocations that happened prior to dynamic initialization will + // cause a crash when free()'d (because Scudo will try and free a GWP-ASan + // allocation). + static gwp_asan::GuardedPoolAllocator GuardedAlloc; #endif // GWP_ASAN_HOOKS StackDepot Depot; @@ -1363,6 +1380,10 @@ } }; +template +gwp_asan::GuardedPoolAllocator + scudo::Allocator::GuardedAlloc; + } // namespace scudo #endif // SCUDO_COMBINED_H_ diff --git a/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp b/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp --- a/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp @@ -94,6 +94,18 @@ EXPECT_EQ(errno, ENOMEM); } +TEST(ScudoWrappersCTest, SmallAlign) { + void *P; + for (size_t Size = 1; Size <= 0x10000; Size <<= 1) { + for (size_t Align = 1; Align <= 0x10000; Align <<= 1) { + for (size_t Count = 0; Count < 3; ++Count) { + P = memalign(Align, Size); + EXPECT_TRUE(reinterpret_cast(P) % Align == 0); + } + } + } +} + TEST(ScudoWrappersCTest, Memalign) { void *P; for (size_t I = FIRST_32_SECOND_64(2U, 3U); I <= 18U; I++) { diff --git a/compiler-rt/lib/scudo/standalone/tests/wrappers_cpp_test.cpp b/compiler-rt/lib/scudo/standalone/tests/wrappers_cpp_test.cpp --- a/compiler-rt/lib/scudo/standalone/tests/wrappers_cpp_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/wrappers_cpp_test.cpp @@ -66,6 +66,10 @@ }; TEST(ScudoWrappersCppTest, New) { + if (getenv("SKIP_TYPE_MISMATCH")) { + printf("Skipped type mismatch tests.\n"); + return; + } testCxxNew(); testCxxNew(); testCxxNew(); diff --git a/compiler-rt/test/scudo/standalone/CMakeLists.txt b/compiler-rt/test/scudo/standalone/CMakeLists.txt --- a/compiler-rt/test/scudo/standalone/CMakeLists.txt +++ b/compiler-rt/test/scudo/standalone/CMakeLists.txt @@ -4,6 +4,12 @@ ${CMAKE_CURRENT_BINARY_DIR}/unit/lit.site.cfg.py) list(APPEND SCUDO_STANDALONE_TEST_DEPS ScudoUnitTests) list(APPEND SCUDO_STANDALONE_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/unit) + if (COMPILER_RT_HAS_GWP_ASAN) + configure_lit_site_cfg( + ${CMAKE_CURRENT_SOURCE_DIR}/unit/gwp_asan/lit.site.cfg.py.in + ${CMAKE_CURRENT_BINARY_DIR}/unit/gwp_asan/lit.site.cfg.py) + list(APPEND SCUDO_STANDALONE_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/unit/gwp_asan) + endif() endif() add_lit_testsuite(check-scudo_standalone diff --git a/compiler-rt/test/scudo/standalone/unit/gwp_asan/lit.site.cfg.py.in b/compiler-rt/test/scudo/standalone/unit/gwp_asan/lit.site.cfg.py.in new file mode 100644 --- /dev/null +++ b/compiler-rt/test/scudo/standalone/unit/gwp_asan/lit.site.cfg.py.in @@ -0,0 +1,24 @@ +@LIT_SITE_CFG_IN_HEADER@ + +# Load common config for all compiler-rt unit tests. +lit_config.load_config(config, "@COMPILER_RT_BINARY_DIR@/unittests/lit.common.unit.configured") + +# Setup config name. +config.name = 'ScudoStandalone-Unit-GwpAsanTorture' + +# Setup test source and exec root. +# For unit tests, we define it as build directory with unit tests. +config.test_exec_root = "@COMPILER_RT_BINARY_DIR@/lib/scudo/standalone/tests" +config.test_source_root = config.test_exec_root + +# This is a second run of the Scudo test suite, but this time under a "torture" +# GWP-ASan mode. Every allocation that can go to GWP-ASan, should go to +# GWP-ASan. This ensures that GWP-ASan allocations meet the same testing +# requirements as the native Scudo allocations. Reserves 409MiB of vaddr space +# for the guarded pool, and this should be paged in on demand. If necessary (for +# 32-bit or places where kernel commits immediately), this could possibly be +# reduced. +config.environment['SCUDO_OPTIONS'] = 'GWP_ASAN_SampleRate=1:GWP_ASAN_MaxSimultaneousAllocations=100000' + +# GWP-ASan doesn't support malloc-type mismatch. +config.environment['SKIP_TYPE_MISMATCH'] = '1'