diff --git a/compiler-rt/CMakeLists.txt b/compiler-rt/CMakeLists.txt --- a/compiler-rt/CMakeLists.txt +++ b/compiler-rt/CMakeLists.txt @@ -45,6 +45,8 @@ mark_as_advanced(COMPILER_RT_BUILD_LIBFUZZER) option(COMPILER_RT_BUILD_PROFILE "Build profile runtime" ON) mark_as_advanced(COMPILER_RT_BUILD_PROFILE) +option(COMPILER_RT_BUILD_HEAPPROF "Build heap profiling runtime" ON) +mark_as_advanced(COMPILER_RT_BUILD_HEAPPROF) option(COMPILER_RT_BUILD_XRAY_NO_PREINIT "Build xray with no preinit patching" OFF) mark_as_advanced(COMPILER_RT_BUILD_XRAY_NO_PREINIT) @@ -67,6 +69,25 @@ -D${COMPILER_RT_ASAN_SHADOW_SCALE_DEFINITION}) endif() +set(COMPILER_RT_HEAPPROF_SHADOW_SCALE "" + CACHE STRING "Override the shadow scale to be used in heapprof runtime") + +if (NOT COMPILER_RT_HEAPPROF_SHADOW_SCALE STREQUAL "") + # Check that the shadow scale value is valid. + if (NOT (COMPILER_RT_HEAPPROF_SHADOW_SCALE GREATER -1 AND + COMPILER_RT_HEAPPROF_SHADOW_SCALE LESS 8)) + message(FATAL_ERROR " + Invalid Heapprof Shadow Scale '${COMPILER_RT_HEAPPROF_SHADOW_SCALE}'.") + endif() + + set(COMPILER_RT_HEAPPROF_SHADOW_SCALE_LLVM_FLAG + -mllvm -heapprof-mapping-scale=${COMPILER_RT_HEAPPROF_SHADOW_SCALE}) + set(COMPILER_RT_HEAPPROF_SHADOW_SCALE_DEFINITION + HEAPPROF_SHADOW_SCALE=${COMPILER_RT_HEAPPROF_SHADOW_SCALE}) + set(COMPILER_RT_HEAPPROF_SHADOW_SCALE_FLAG + -D${COMPILER_RT_HEAPPROF_SHADOW_SCALE_DEFINITION}) +endif() + set(COMPILER_RT_HWASAN_WITH_INTERCEPTORS ON CACHE BOOL "Enable libc interceptors in HWASan (testing mode)") diff --git a/compiler-rt/cmake/config-ix.cmake b/compiler-rt/cmake/config-ix.cmake --- a/compiler-rt/cmake/config-ix.cmake +++ b/compiler-rt/cmake/config-ix.cmake @@ -323,6 +323,7 @@ endif() set(ALL_MSAN_SUPPORTED_ARCH ${X86_64} ${MIPS64} ${ARM64} ${PPC64} ${S390X}) set(ALL_HWASAN_SUPPORTED_ARCH ${X86_64} ${ARM64}) +set(ALL_HEAPPROF_SUPPORTED_ARCH ${X86_64}) set(ALL_PROFILE_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${PPC64} ${MIPS32} ${MIPS64} ${S390X} ${SPARC} ${SPARCV9}) set(ALL_TSAN_SUPPORTED_ARCH ${X86_64} ${MIPS64} ${ARM64} ${PPC64}) @@ -550,6 +551,9 @@ list_intersect(HWASAN_SUPPORTED_ARCH ALL_HWASAN_SUPPORTED_ARCH SANITIZER_COMMON_SUPPORTED_ARCH) + list_intersect(HEAPPROF_SUPPORTED_ARCH + ALL_HEAPPROF_SUPPORTED_ARCH + SANITIZER_COMMON_SUPPORTED_ARCH) list_intersect(PROFILE_SUPPORTED_ARCH ALL_PROFILE_SUPPORTED_ARCH SANITIZER_COMMON_SUPPORTED_ARCH) @@ -598,6 +602,7 @@ filter_available_targets(LSAN_SUPPORTED_ARCH ${ALL_LSAN_SUPPORTED_ARCH}) filter_available_targets(MSAN_SUPPORTED_ARCH ${ALL_MSAN_SUPPORTED_ARCH}) filter_available_targets(HWASAN_SUPPORTED_ARCH ${ALL_HWASAN_SUPPORTED_ARCH}) + filter_available_targets(HEAPPROF_SUPPORTED_ARCH ${ALL_HEAPPROF_SUPPORTED_ARCH}) filter_available_targets(PROFILE_SUPPORTED_ARCH ${ALL_PROFILE_SUPPORTED_ARCH}) filter_available_targets(TSAN_SUPPORTED_ARCH ${ALL_TSAN_SUPPORTED_ARCH}) filter_available_targets(UBSAN_SUPPORTED_ARCH ${ALL_UBSAN_SUPPORTED_ARCH}) @@ -701,6 +706,13 @@ set(COMPILER_RT_HAS_HWASAN FALSE) endif() +if (COMPILER_RT_HAS_SANITIZER_COMMON AND HEAPPROF_SUPPORTED_ARCH AND + OS_NAME MATCHES "Linux") + set(COMPILER_RT_HAS_HEAPPROF TRUE) +else() + set(COMPILER_RT_HAS_HEAPPROF FALSE) +endif() + if (PROFILE_SUPPORTED_ARCH AND NOT LLVM_USE_SANITIZER AND OS_NAME MATCHES "Darwin|Linux|FreeBSD|Windows|Android|Fuchsia|SunOS|NetBSD") set(COMPILER_RT_HAS_PROFILE TRUE) diff --git a/compiler-rt/include/CMakeLists.txt b/compiler-rt/include/CMakeLists.txt --- a/compiler-rt/include/CMakeLists.txt +++ b/compiler-rt/include/CMakeLists.txt @@ -5,6 +5,7 @@ sanitizer/common_interface_defs.h sanitizer/coverage_interface.h sanitizer/dfsan_interface.h + sanitizer/heapprof_interface.h sanitizer/hwasan_interface.h sanitizer/linux_syscall_hooks.h sanitizer/lsan_interface.h diff --git a/compiler-rt/include/sanitizer/heapprof_interface.h b/compiler-rt/include/sanitizer/heapprof_interface.h new file mode 100644 --- /dev/null +++ b/compiler-rt/include/sanitizer/heapprof_interface.h @@ -0,0 +1,60 @@ +//===-- sanitizer/heapprof_interface.h --------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler (HeapProf). +// +// Public interface header. +//===----------------------------------------------------------------------===// +#ifndef SANITIZER_HEAPPROF_INTERFACE_H +#define SANITIZER_HEAPPROF_INTERFACE_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif +/// Records access to a memory region ([addr, addr+size)). +/// +/// This memory must be previously allocated by your program. +/// +/// \param addr Start of memory region. +/// \param size Size of memory region. +void __heapprof_record_access_range(void const volatile *addr, size_t size); + +/// Records access to a memory address addr. +/// +/// This memory must be previously allocated by your program. +/// +/// \param addr Accessed memory address +void __heapprof_record_access(void const volatile *addr); + +/// User-provided callback on HeapProf errors. +/// +/// You can provide a function that would be called immediately when HeapProf +/// detects an error. This is useful in cases when HeapProf detects an error but +/// your program crashes before the HeapProf report is printed. +void __heapprof_on_error(void); + +/// Prints accumulated statistics to stderr (useful for calling from the +/// debugger). +void __heapprof_print_accumulated_stats(void); + +/// User-provided default option settings. +/// +/// You can provide your own implementation of this function to return a string +/// containing HeapProf runtime options (for example, +/// verbosity=1:print_stats=1). +/// +/// \returns Default options string. +const char *__heapprof_default_options(void); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // SANITIZER_HEAPPROF_INTERFACE_H diff --git a/compiler-rt/lib/CMakeLists.txt b/compiler-rt/lib/CMakeLists.txt --- a/compiler-rt/lib/CMakeLists.txt +++ b/compiler-rt/lib/CMakeLists.txt @@ -60,6 +60,10 @@ compiler_rt_build_runtime(fuzzer) endif() +if(COMPILER_RT_BUILD_HEAPPROF AND COMPILER_RT_HAS_SANITIZER_COMMON) + compiler_rt_build_runtime(heapprof) +endif() + # It doesn't normally make sense to build runtimes when a sanitizer is enabled, # so we don't add_subdirectory the runtimes in that case. However, the opposite # is true for fuzzers that exercise parts of the runtime. So we add the fuzzer diff --git a/compiler-rt/lib/heapprof/CMakeLists.txt b/compiler-rt/lib/heapprof/CMakeLists.txt new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/CMakeLists.txt @@ -0,0 +1,200 @@ +# Build for the Heap Profiler runtime support library. + +set(HEAPPROF_SOURCES + heapprof_allocator.cpp + heapprof_descriptions.cpp + heapprof_errors.cpp + heapprof_flags.cpp + heapprof_interceptors.cpp + heapprof_interceptors_memintrinsics.cpp + heapprof_linux.cpp + heapprof_malloc_linux.cpp + heapprof_posix.cpp + heapprof_report.cpp + heapprof_rtl.cpp + heapprof_shadow_setup.cpp + heapprof_stack.cpp + heapprof_stats.cpp + heapprof_thread.cpp + ) + +set(HEAPPROF_CXX_SOURCES + heapprof_new_delete.cpp + ) + +set(HEAPPROF_PREINIT_SOURCES + heapprof_preinit.cpp + ) + +SET(HEAPPROF_HEADERS + heapprof_allocator.h + heapprof_descriptions.h + heapprof_errors.h + heapprof_flags.h + heapprof_flags.inc + heapprof_init_version.h + heapprof_interceptors.h + heapprof_interceptors_memintrinsics.h + heapprof_interface.inc + heapprof_interface_internal.h + heapprof_internal.h + heapprof_mapping.h + heapprof_report.h + heapprof_stack.h + heapprof_stats.h + heapprof_thread.h + ) + +include_directories(..) + +set(HEAPPROF_CFLAGS ${SANITIZER_COMMON_CFLAGS}) +set(HEAPPROF_COMMON_DEFINITIONS ${COMPILER_RT_HEAPPROF_SHADOW_SCALE_DEFINITION}) + +append_rtti_flag(OFF HEAPPROF_CFLAGS) + +set(HEAPPROF_DYNAMIC_LINK_FLAGS ${SANITIZER_COMMON_LINK_FLAGS}) + +set(HEAPPROF_DYNAMIC_DEFINITIONS + ${HEAPPROF_COMMON_DEFINITIONS} HEAPPROF_DYNAMIC=1) + +set(HEAPPROF_DYNAMIC_CFLAGS ${HEAPPROF_CFLAGS}) +append_list_if(COMPILER_RT_HAS_FTLS_MODEL_INITIAL_EXEC + -ftls-model=initial-exec HEAPPROF_DYNAMIC_CFLAGS) + +set(HEAPPROF_DYNAMIC_LIBS ${SANITIZER_CXX_ABI_LIBRARIES} ${SANITIZER_COMMON_LINK_LIBS}) + +append_list_if(COMPILER_RT_HAS_LIBDL dl HEAPPROF_DYNAMIC_LIBS) +append_list_if(COMPILER_RT_HAS_LIBRT rt HEAPPROF_DYNAMIC_LIBS) +append_list_if(COMPILER_RT_HAS_LIBM m HEAPPROF_DYNAMIC_LIBS) +append_list_if(COMPILER_RT_HAS_LIBPTHREAD pthread HEAPPROF_DYNAMIC_LIBS) +append_list_if(COMPILER_RT_HAS_LIBLOG log HEAPPROF_DYNAMIC_LIBS) + +if (TARGET cxx-headers OR HAVE_LIBCXX) + set(HEAPPROF_DEPS cxx-headers) +endif() + +# Compile HeapProf sources into an object library. + +add_compiler_rt_object_libraries(RTHeapprof_dynamic + OS ${SANITIZER_COMMON_SUPPORTED_OS} + ARCHS ${HEAPPROF_SUPPORTED_ARCH} + SOURCES ${HEAPPROF_SOURCES} ${HEAPPROF_CXX_SOURCES} + ADDITIONAL_HEADERS ${HEAPPROF_HEADERS} + CFLAGS ${HEAPPROF_DYNAMIC_CFLAGS} + DEFS ${HEAPPROF_DYNAMIC_DEFINITIONS} + DEPS ${HEAPPROF_DEPS}) + + add_compiler_rt_object_libraries(RTHeapprof + ARCHS ${HEAPPROF_SUPPORTED_ARCH} + SOURCES ${HEAPPROF_SOURCES} + ADDITIONAL_HEADERS ${HEAPPROF_HEADERS} + CFLAGS ${HEAPPROF_CFLAGS} + DEFS ${HEAPPROF_COMMON_DEFINITIONS} + DEPS ${HEAPPROF_DEPS}) + add_compiler_rt_object_libraries(RTHeapprof_cxx + ARCHS ${HEAPPROF_SUPPORTED_ARCH} + SOURCES ${HEAPPROF_CXX_SOURCES} + ADDITIONAL_HEADERS ${HEAPPROF_HEADERS} + CFLAGS ${HEAPPROF_CFLAGS} + DEFS ${HEAPPROF_COMMON_DEFINITIONS} + DEPS ${HEAPPROF_DEPS}) + add_compiler_rt_object_libraries(RTHeapprof_preinit + ARCHS ${HEAPPROF_SUPPORTED_ARCH} + SOURCES ${HEAPPROF_PREINIT_SOURCES} + ADDITIONAL_HEADERS ${HEAPPROF_HEADERS} + CFLAGS ${HEAPPROF_CFLAGS} + DEFS ${HEAPPROF_COMMON_DEFINITIONS} + DEPS ${HEAPPROF_DEPS}) + + file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp "") + add_compiler_rt_object_libraries(RTHeapprof_dynamic_version_script_dummy + ARCHS ${HEAPPROF_SUPPORTED_ARCH} + SOURCES ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp + CFLAGS ${HEAPPROF_DYNAMIC_CFLAGS} + DEFS ${HEAPPROF_DYNAMIC_DEFINITIONS} + DEPS ${HEAPPROF_DEPS}) + +# Build HeapProf runtimes shipped with Clang. +add_compiler_rt_component(heapprof) + + # Build separate libraries for each target. + + set(HEAPPROF_COMMON_RUNTIME_OBJECT_LIBS + RTInterception + RTSanitizerCommon + RTSanitizerCommonLibc + RTSanitizerCommonCoverage + RTSanitizerCommonSymbolizer) + + add_compiler_rt_runtime(clang_rt.heapprof + STATIC + ARCHS ${HEAPPROF_SUPPORTED_ARCH} + OBJECT_LIBS RTHeapprof_preinit + RTHeapprof + ${HEAPPROF_COMMON_RUNTIME_OBJECT_LIBS} + CFLAGS ${HEAPPROF_CFLAGS} + DEFS ${HEAPPROF_COMMON_DEFINITIONS} + PARENT_TARGET heapprof) + + add_compiler_rt_runtime(clang_rt.heapprof_cxx + STATIC + ARCHS ${HEAPPROF_SUPPORTED_ARCH} + OBJECT_LIBS RTHeapprof_cxx + CFLAGS ${HEAPPROF_CFLAGS} + DEFS ${HEAPPROF_COMMON_DEFINITIONS} + PARENT_TARGET heapprof) + + add_compiler_rt_runtime(clang_rt.heapprof-preinit + STATIC + ARCHS ${HEAPPROF_SUPPORTED_ARCH} + OBJECT_LIBS RTHeapprof_preinit + CFLAGS ${HEAPPROF_CFLAGS} + DEFS ${HEAPPROF_COMMON_DEFINITIONS} + PARENT_TARGET heapprof) + + foreach(arch ${HEAPPROF_SUPPORTED_ARCH}) + if (UNIX) + add_sanitizer_rt_version_list(clang_rt.heapprof-dynamic-${arch} + LIBS clang_rt.heapprof-${arch} clang_rt.heapprof_cxx-${arch} + EXTRA heapprof.syms.extra) + set(VERSION_SCRIPT_FLAG + -Wl,--version-script,${CMAKE_CURRENT_BINARY_DIR}/clang_rt.heapprof-dynamic-${arch}.vers) + set_property(SOURCE + ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp + APPEND PROPERTY + OBJECT_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/clang_rt.heapprof-dynamic-${arch}.vers) + else() + set(VERSION_SCRIPT_FLAG) + endif() + + set(HEAPPROF_DYNAMIC_WEAK_INTERCEPTION) + + add_compiler_rt_runtime(clang_rt.heapprof + SHARED + ARCHS ${arch} + OBJECT_LIBS ${HEAPPROF_COMMON_RUNTIME_OBJECT_LIBS} + RTHeapprof_dynamic + # The only purpose of RTHeapprof_dynamic_version_script_dummy is to + # carry a dependency of the shared runtime on the version script. + # Replacing it with a straightforward + # add_dependencies(clang_rt.heapprof-dynamic-${arch} clang_rt.heapprof-dynamic-${arch}-version-list) + # generates an order-only dependency in ninja. + RTHeapprof_dynamic_version_script_dummy + ${HEAPPROF_DYNAMIC_WEAK_INTERCEPTION} + CFLAGS ${HEAPPROF_DYNAMIC_CFLAGS} + LINK_FLAGS ${HEAPPROF_DYNAMIC_LINK_FLAGS} + ${VERSION_SCRIPT_FLAG} + LINK_LIBS ${HEAPPROF_DYNAMIC_LIBS} + DEFS ${HEAPPROF_DYNAMIC_DEFINITIONS} + PARENT_TARGET heapprof) + + if (SANITIZER_USE_SYMBOLS) + add_sanitizer_rt_symbols(clang_rt.heapprof_cxx + ARCHS ${arch}) + add_dependencies(heapprof clang_rt.heapprof_cxx-${arch}-symbols) + add_sanitizer_rt_symbols(clang_rt.heapprof + ARCHS ${arch} + EXTRA heapprof.syms.extra) + add_dependencies(heapprof clang_rt.heapprof-${arch}-symbols) + endif() + endforeach() diff --git a/compiler-rt/lib/heapprof/README.txt b/compiler-rt/lib/heapprof/README.txt new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/README.txt @@ -0,0 +1,17 @@ +HeapProfiling RT +================================ +This directory contains sources of the HeapProfiling (HeapProf) runtime library. + +Directory structure: +README.txt : This file. +CMakeLists.txt : File for cmake-based build. +heapprof_*.{cc,h} : Sources of the heapprof runtime library. + +Also HeapProf runtime needs the following libraries: +lib/interception/ : Machinery used to intercept function calls. +lib/sanitizer_common/ : Code shared between various sanitizers. + +HeapProf runtime can only be built by CMake. You can run HeapProf tests +from the root of your CMake build tree: + +make check-heapprof diff --git a/compiler-rt/lib/heapprof/heapprof.syms.extra b/compiler-rt/lib/heapprof/heapprof.syms.extra new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof.syms.extra @@ -0,0 +1 @@ +__heapprof_* diff --git a/compiler-rt/lib/heapprof/heapprof_allocator.h b/compiler-rt/lib/heapprof/heapprof_allocator.h new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_allocator.h @@ -0,0 +1,125 @@ +//===-- heapprof_allocator.h ------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// HeapProf-private header for heapprof_allocator.cpp. +//===----------------------------------------------------------------------===// + +#ifndef HEAPPROF_ALLOCATOR_H +#define HEAPPROF_ALLOCATOR_H + +#include "heapprof_flags.h" +#include "heapprof_interceptors.h" +#include "heapprof_internal.h" +#include "sanitizer_common/sanitizer_allocator.h" +#include "sanitizer_common/sanitizer_list.h" + +#if !defined(__x86_64__) +#error Unsupported platform +#endif + +namespace __heapprof { + +enum AllocType { + FROM_MALLOC = 1, // Memory block came from malloc, calloc, realloc, etc. + FROM_NEW = 2, // Memory block came from operator new. + FROM_NEW_BR = 3 // Memory block came from operator new [ ] +}; + +struct HeapprofChunk; + +void InitializeAllocator(); + +struct HeapprofMapUnmapCallback { + void OnMap(uptr p, uptr size) const; + void OnUnmap(uptr p, uptr size) const; +}; + +#if SANITIZER_CAN_USE_ALLOCATOR64 +const uptr kAllocatorSpace = 0x600000000000ULL; +const uptr kAllocatorSize = 0x40000000000ULL; // 4T. +typedef DefaultSizeClassMap SizeClassMap; +template +struct AP64 { // Allocator64 parameters. Deliberately using a short name. + static const uptr kSpaceBeg = kAllocatorSpace; + static const uptr kSpaceSize = kAllocatorSize; + static const uptr kMetadataSize = 0; + typedef __heapprof::SizeClassMap SizeClassMap; + typedef HeapprofMapUnmapCallback MapUnmapCallback; + static const uptr kFlags = 0; + using AddressSpaceView = AddressSpaceViewTy; +}; + +template +using PrimaryAllocatorASVT = SizeClassAllocator64>; +using PrimaryAllocator = PrimaryAllocatorASVT; +#else // Fallback to SizeClassAllocator32. +typedef CompactSizeClassMap SizeClassMap; +template struct AP32 { + static const uptr kSpaceBeg = 0; + static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; + static const uptr kMetadataSize = 16; + typedef __heapprof::SizeClassMap SizeClassMap; + static const uptr kRegionSizeLog = 20; + using AddressSpaceView = AddressSpaceViewTy; + typedef HeapprofMapUnmapCallback MapUnmapCallback; + static const uptr kFlags = 0; +}; +template +using PrimaryAllocatorASVT = SizeClassAllocator32>; +using PrimaryAllocator = PrimaryAllocatorASVT; +#endif // SANITIZER_CAN_USE_ALLOCATOR64 + +static const uptr kNumberOfSizeClasses = SizeClassMap::kNumClasses; + +template +using HeapprofAllocatorASVT = + CombinedAllocator>; +using HeapprofAllocator = HeapprofAllocatorASVT; +using AllocatorCache = HeapprofAllocator::AllocatorCache; + +struct HeapprofThreadLocalMallocStorage { + uptr quarantine_cache[16]; + AllocatorCache allocator_cache; + void CommitBack(); + +private: + // These objects are allocated via mmap() and are zero-initialized. + HeapprofThreadLocalMallocStorage() {} +}; + +void *heapprof_memalign(uptr alignment, uptr size, BufferedStackTrace *stack, + AllocType alloc_type); +void heapprof_free(void *ptr, BufferedStackTrace *stack, AllocType alloc_type); +void heapprof_delete(void *ptr, uptr size, uptr alignment, + BufferedStackTrace *stack, AllocType alloc_type); + +void *heapprof_malloc(uptr size, BufferedStackTrace *stack); +void *heapprof_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack); +void *heapprof_realloc(void *p, uptr size, BufferedStackTrace *stack); +void *heapprof_reallocarray(void *p, uptr nmemb, uptr size, + BufferedStackTrace *stack); +void *heapprof_valloc(uptr size, BufferedStackTrace *stack); +void *heapprof_pvalloc(uptr size, BufferedStackTrace *stack); + +void *heapprof_aligned_alloc(uptr alignment, uptr size, + BufferedStackTrace *stack); +int heapprof_posix_memalign(void **memptr, uptr alignment, uptr size, + BufferedStackTrace *stack); +uptr heapprof_malloc_usable_size(const void *ptr, uptr pc, uptr bp); + +uptr heapprof_mz_size(const void *ptr); +void heapprof_mz_force_lock(); +void heapprof_mz_force_unlock(); + +void PrintInternalAllocatorStats(); +void HeapprofSoftRssLimitExceededCallback(bool exceeded); + +} // namespace __heapprof +#endif // HEAPPROF_ALLOCATOR_H diff --git a/compiler-rt/lib/heapprof/heapprof_allocator.cpp b/compiler-rt/lib/heapprof/heapprof_allocator.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_allocator.cpp @@ -0,0 +1,869 @@ +//===-- heapprof_allocator.cpp --------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// Implementation of HeapProf's memory allocator, which uses the allocator +// from sanitizer_common. +// +//===----------------------------------------------------------------------===// + +#include "heapprof_allocator.h" +#include "heapprof_mapping.h" +#include "heapprof_report.h" +#include "heapprof_stack.h" +#include "heapprof_thread.h" +#include "sanitizer_common/sanitizer_allocator_checks.h" +#include "sanitizer_common/sanitizer_allocator_interface.h" +#include +#include +#include +#include +#undef errno +#include "sanitizer_common/sanitizer_errno.h" +#include "sanitizer_common/sanitizer_file.h" +#include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_internal_defs.h" +#include "sanitizer_common/sanitizer_list.h" +#include "sanitizer_common/sanitizer_stackdepot.h" + +namespace __heapprof { + +static int GetCpuId(void) { + // _heapprof_preinit is called via the preinit_array, which subsequently calls + // malloc. Since this is before _dl_init calls VDSO_SETUP, sched_getcpu + // will seg fault as the address of __vdso_getcpu will be null. + if (!heapprof_init_done) + return -1; + return sched_getcpu(); +} + +// Compute the timestamp in ms. +static int GetTimestamp(void) { + // timespec_get will segfault if called from dl_init + if (!heapprof_timestamp_inited) { + // By returning 0, this will be effectively treated as being + // timestamped at heapprof init time (when heapprof_init_timestamp_s + // is initialized). + return 0; + } + timespec ts; + timespec_get(&ts, TIME_UTC); + return (ts.tv_sec - heapprof_init_timestamp_s) * 1000 + ts.tv_nsec / 1000; +} + +static HeapprofAllocator &get_allocator(); + +// The memory chunk allocated from the underlying allocator looks like this: +// H H U U U U U U +// H -- ChunkHeader (32 bytes) +// U -- user memory. + +// If there is left padding before the ChunkHeader (due to use of memalign), +// we store a magic value in the first uptr word of the memory block and +// store the address of ChunkHeader in the next uptr. +// M B L L L L L L L L L H H U U U U U U +// | ^ +// ---------------------| +// M -- magic value kAllocBegMagic +// B -- address of ChunkHeader pointing to the first 'H' +static const uptr kAllocBegMagic = 0xCC6E96B9; + +// Should be no more than 32-bytes +struct ChunkHeader { + // 1-st 4 bytes. + u32 from_memalign : 1; + // This field is used for small sizes. For large sizes it is equal to + // SizeClassMap::kMaxSize and the actual size is stored in the + // SecondaryAllocator's metadata. + u32 user_requested_size : 29; + // 2 bits available + // 2-nd 4 bytes + u32 cpu_id; + // 3-rd 4 bytes + u32 timestamp_ms; + // 4-th 4 bytes available + u32 dummy; + // 5-th and 6-th 4 bytes + u64 alloc_context_id; + // 7-th and 8-th 4 bytes + u64 data_type_id; // TODO: hash of type name +}; + +static const uptr kChunkHeaderSize = sizeof(ChunkHeader); +COMPILER_CHECK(kChunkHeaderSize == 32); + +struct HeapprofChunk : ChunkHeader { + uptr Beg() { return reinterpret_cast(this) + kChunkHeaderSize; } + uptr UsedSize() { + if (user_requested_size != SizeClassMap::kMaxSize) + return user_requested_size; + return *reinterpret_cast(get_allocator().GetMetaData(AllocBeg())); + } + void *AllocBeg() { + if (from_memalign) + return get_allocator().GetBlockBegin(reinterpret_cast(this)); + return reinterpret_cast(Beg() - kChunkHeaderSize); + } +}; + +void FlushUnneededHeapProfShadowMemory(uptr p, uptr size) { + // Since heapprof's mapping is compacting, the shadow chunk may be + // not page-aligned, so we only flush the page-aligned portion. + ReleaseMemoryPagesToOS(MemToShadow(p), MemToShadow(p + size)); +} + +void HeapprofMapUnmapCallback::OnMap(uptr p, uptr size) const { + // Statistics. + HeapprofStats &thread_stats = GetCurrentThreadStats(); + thread_stats.mmaps++; + thread_stats.mmaped += size; +} +void HeapprofMapUnmapCallback::OnUnmap(uptr p, uptr size) const { + // We are about to unmap a chunk of user memory. + // Mark the corresponding shadow memory as not needed. + FlushUnneededHeapProfShadowMemory(p, size); + // Statistics. + HeapprofStats &thread_stats = GetCurrentThreadStats(); + thread_stats.munmaps++; + thread_stats.munmaped += size; +} + +AllocatorCache *GetAllocatorCache(HeapprofThreadLocalMallocStorage *ms) { + CHECK(ms); + return &ms->allocator_cache; +} + +struct HeapInfoBlock { + u32 alloc_count; + u64 total_access_count, min_access_count, max_access_count; + u64 total_size; + u32 min_size, max_size; + u32 alloc_timestamp, dealloc_timestamp; + u64 total_lifetime; + u32 min_lifetime, max_lifetime; + u32 alloc_cpu_id, dealloc_cpu_id; + u32 num_migrated_cpu; + + // Only compared to prior deallocated object currently. + u32 num_lifetime_overlaps; + u32 num_same_alloc_cpu; + u32 num_same_dealloc_cpu; + + u64 data_type_id; // TODO: hash of type name + + HeapInfoBlock() : alloc_count(0) {} + + HeapInfoBlock(u32 size, u64 access_count, u32 alloc_timestamp, + u32 dealloc_timestamp, u32 alloc_cpu, u32 dealloc_cpu) + : alloc_count(1), total_access_count(access_count), + min_access_count(access_count), max_access_count(access_count), + total_size(size), min_size(size), max_size(size), + alloc_timestamp(alloc_timestamp), dealloc_timestamp(dealloc_timestamp), + total_lifetime(alloc_timestamp ? (dealloc_timestamp - alloc_timestamp) + : 0), + min_lifetime(total_lifetime), max_lifetime(total_lifetime), + alloc_cpu_id(alloc_cpu), dealloc_cpu_id(dealloc_cpu), + num_lifetime_overlaps(0), num_same_alloc_cpu(0), + num_same_dealloc_cpu(0) { + num_migrated_cpu = alloc_cpu_id != dealloc_cpu_id; + } + + void Print(u64 id) { + char buffer[100]; + if (flags()->print_terse) { + sprintf(buffer, "%.2f", ((float)total_size) / alloc_count); + Printf("HIB:%llu/%u/%s/%u/%u/", id, alloc_count, buffer, min_size, + max_size); + sprintf(buffer, "%.2f", ((float)total_access_count) / alloc_count); + Printf("%s/%u/%u/", buffer, min_access_count, max_access_count); + sprintf(buffer, "%.2f", ((float)total_lifetime) / alloc_count); + Printf("%s/%u/%u/", buffer, min_lifetime, max_lifetime); + Printf("%u/%u/%u/%u\n", num_migrated_cpu, num_lifetime_overlaps, + num_same_alloc_cpu, num_same_dealloc_cpu); + } else { + sprintf(buffer, "%.2f", ((float)total_size) / alloc_count); + Printf("Heap allocation stack id = %llu\n", id); + Printf("\talloc_count %u, size (ave/min/max) %s / %u / %u\n", alloc_count, + buffer, min_size, max_size); + sprintf(buffer, "%.2f", ((float)total_access_count) / alloc_count); + Printf("\taccess_count (ave/min/max): %s / %u / %u\n", buffer, + min_access_count, max_access_count); + sprintf(buffer, "%.2f", ((float)total_lifetime) / alloc_count); + Printf("\tlifetime (ave/min/max): %s / %u / %u\n", buffer, min_lifetime, + max_lifetime); + Printf("\tnum migrated: %u, num lifetime overlaps: %u, num same alloc " + "cpu: %u, num same dealloc_cpu: %u\n", + num_migrated_cpu, num_lifetime_overlaps, num_same_alloc_cpu, + num_same_dealloc_cpu); + } + } + + static void printHeader() { + CHECK(flags()->print_terse); + Printf("HIB:StackID/AllocCount/AveSize/MinSize/MaxSize/AveAccessCount/" + "MinAccessCount/MaxAccessCount/AveLifetime/MinLifetime/MaxLifetime/" + "NumMigratedCpu/NumLifetimeOverlaps/NumSameAllocCpu/" + "NumSameDeallocCpu\n"); + } + + void Merge(HeapInfoBlock &newHIB) { + alloc_count += newHIB.alloc_count; + + total_access_count += newHIB.total_access_count; + min_access_count = Min(min_access_count, newHIB.min_access_count); + max_access_count = Max(max_access_count, newHIB.max_access_count); + + total_size += newHIB.total_size; + min_size = Min(min_size, newHIB.min_size); + max_size = Max(max_size, newHIB.max_size); + + total_lifetime += newHIB.total_lifetime; + min_lifetime = Min(min_lifetime, newHIB.min_lifetime); + max_lifetime = Max(max_lifetime, newHIB.max_lifetime); + + // We know newHIB was deallocated later, so just need to check if it was + // allocated before last one deallocated. + num_lifetime_overlaps += newHIB.alloc_timestamp < dealloc_timestamp; + alloc_timestamp = newHIB.alloc_timestamp; + dealloc_timestamp = newHIB.dealloc_timestamp; + + num_same_alloc_cpu += alloc_cpu_id == newHIB.alloc_cpu_id; + num_same_dealloc_cpu += dealloc_cpu_id == newHIB.dealloc_cpu_id; + alloc_cpu_id = newHIB.alloc_cpu_id; + dealloc_cpu_id = newHIB.dealloc_cpu_id; + } +}; + +static u32 AccessCount = 0; +static u32 MissCount = 0; + +struct SetEntry { + SetEntry() : id(0), HIB() {} + bool Empty() { return id == 0; } + void Print() { + CHECK(!Empty()); + HIB.Print(id); + } + // The stack id + u64 id; + HeapInfoBlock HIB; +}; + +struct CacheSet { + enum { kSetSize = 4 }; + + void PrintAll() { + for (int i = 0; i < kSetSize; i++) { + if (Entries[i].Empty()) + continue; + Entries[i].Print(); + } + } + void insertOrMerge(u64 new_id, HeapInfoBlock &newHIB) { + AccessCount++; + SetAccessCount++; + + for (int i = 0; i < kSetSize; i++) { + auto id = Entries[i].id; + // Check if this is a hit or an empty entry. Since we always move any + // filled locations to the front of the array (see below), we don't need + // to look after finding the first empty entry. + if (id == new_id || !id) { + if (id == 0) { + Entries[i].id = new_id; + Entries[i].HIB = newHIB; + } else { + Entries[i].HIB.Merge(newHIB); + } + // Assuming some id locality, we try to swap the matching entry + // into the first set position. + if (i != 0) { + auto tmp = Entries[0]; + Entries[0] = Entries[i]; + Entries[i] = tmp; + } + return; + } + } + + // Miss + MissCount++; + SetMissCount++; + + // We try to find the entries with the lowest alloc count to be evicted: + int min_idx = 0; + u64 min_count = Entries[0].HIB.alloc_count; + for (int i = 1; i < kSetSize; i++) { + CHECK(!Entries[i].Empty()); + if (Entries[i].HIB.alloc_count < min_count) { + min_idx = i; + min_count = Entries[i].HIB.alloc_count; + } + } + + // Print the evicted entry profile information + if (!flags()->print_terse) + Printf("Evicted:\n"); + Entries[min_idx].Print(); + + // Similar to the hit case, put new HIB in first set position. + if (min_idx != 0) + Entries[min_idx] = Entries[0]; + Entries[0].id = new_id; + Entries[0].HIB = newHIB; + } + + void PrintMissRate(int i) { + char buffer[100]; + sprintf(buffer, "%5.2f%%", + SetAccessCount ? SetMissCount * 100.0 / SetAccessCount : 0.0); + Printf("Set %d miss rate: %d / %d = %s\n", i, SetMissCount, SetAccessCount, + buffer); + } + + SetEntry Entries[kSetSize]; + u32 SetAccessCount = 0; + u32 SetMissCount = 0; +}; + +struct HeapInfoBlockCache { + HeapInfoBlockCache() { + if (flags()->dump_process_map) + DumpProcessMap(); + if (flags()->print_terse) + HeapInfoBlock::printHeader(); + Sets = + (CacheSet *)malloc(sizeof(CacheSet) * flags()->heap_info_cache_entries); + } + + ~HeapInfoBlockCache() { free(Sets); } + + void insertOrMerge(u64 new_id, HeapInfoBlock &newHIB) { + u64 hv = new_id; + + // Use mod method where number of entries should be a prime close to power + // of 2. + hv %= flags()->heap_info_cache_entries; + + return Sets[hv].insertOrMerge(new_id, newHIB); + } + + void PrintAll() { + for (int i = 0; i < flags()->heap_info_cache_entries; i++) { + Sets[i].PrintAll(); + } + } + + void PrintMissRate() { + if (!flags()->print_heap_info_cache_miss_rate) + return; + char buffer[100]; + sprintf(buffer, "%5.2f%%", + AccessCount ? MissCount * 100.0 / AccessCount : 0.0); + Printf("Overall miss rate: %d / %d = %s\n", MissCount, AccessCount, buffer); + if (flags()->print_heap_info_cache_miss_rate_details) + for (int i = 0; i < flags()->heap_info_cache_entries; i++) + Sets[i].PrintMissRate(i); + } + + CacheSet *Sets; +}; + +// Accumulates the access count from the shadow for the given pointer and size. +u64 GetShadowCount(uptr p, u32 size) { + u64 *shadow = (u64 *)MEM_TO_SHADOW(p); + u64 *shadow_end = (u64 *)MEM_TO_SHADOW(p + size); + u64 count = 0; + for (; shadow <= shadow_end; shadow++) + count += *shadow; + return count; +} + +// Clears the shadow counters (when memory is allocated). +void ClearShadow(uptr addr, uptr size) { + CHECK(AddrIsAlignedByGranularity(addr)); + CHECK(AddrIsInMem(addr)); + CHECK(AddrIsAlignedByGranularity(addr + size)); + CHECK(AddrIsInMem(addr + size - SHADOW_GRANULARITY)); + CHECK(REAL(memset)); + uptr shadow_beg = MEM_TO_SHADOW(addr); + uptr shadow_end = MEM_TO_SHADOW(addr + size - SHADOW_GRANULARITY) + 1; + if (shadow_end - shadow_beg < common_flags()->clear_shadow_mmap_threshold) { + REAL(memset)((void *)shadow_beg, 0, shadow_end - shadow_beg); + } else { + uptr page_size = GetPageSizeCached(); + uptr page_beg = RoundUpTo(shadow_beg, page_size); + uptr page_end = RoundDownTo(shadow_end, page_size); + + if (page_beg >= page_end) { + REAL(memset)((void *)shadow_beg, 0, shadow_end - shadow_beg); + } else { + if (page_beg != shadow_beg) { + REAL(memset)((void *)shadow_beg, 0, page_beg - shadow_beg); + } + if (page_end != shadow_end) { + REAL(memset)((void *)page_end, 0, shadow_end - page_end); + } + ReserveShadowMemoryRange(page_beg, page_end - 1, nullptr); + } + } +} + +struct Allocator { + static const uptr kMaxAllowedMallocSize = + FIRST_32_SECOND_64(3UL << 30, 1ULL << 40); + + HeapprofAllocator allocator; + StaticSpinMutex fallback_mutex; + AllocatorCache fallback_allocator_cache; + + uptr max_user_defined_malloc_size; + atomic_uint8_t rss_limit_exceeded; + + HeapInfoBlockCache HeapInfoBlockTable; + bool destructing; + + // ------------------- Initialization ------------------------ + explicit Allocator(LinkerInitialized) : destructing(false) {} + + ~Allocator() { FinishAndPrint(); } + + void FinishAndPrint() { + if (!flags()->print_terse) + Printf("Live on exit:\n"); + allocator.ForceLock(); + allocator.ForEachChunk( + [](uptr chunk, void *alloc) { + HeapprofChunk *m = + ((Allocator *)alloc)->GetHeapprofChunk((void *)chunk); + // The size is reset to 0 on deallocation (and a min of 1 on + // allocation). + if (!m->user_requested_size) + return; + uptr user_beg = ((uptr)m) + kChunkHeaderSize; + u64 c = GetShadowCount(user_beg, m->user_requested_size); + long curtime = GetTimestamp(); + HeapInfoBlock newHIB(m->user_requested_size, c, m->timestamp_ms, + curtime, m->cpu_id, GetCpuId()); + ((Allocator *)alloc) + ->HeapInfoBlockTable.insertOrMerge(m->alloc_context_id, newHIB); + }, + this); + allocator.ForceUnlock(); + + destructing = true; + HeapInfoBlockTable.PrintMissRate(); + HeapInfoBlockTable.PrintAll(); + StackDepotPrintAll(); + } + + void InitLinkerInitialized() { + SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null); + allocator.InitLinkerInitialized( + common_flags()->allocator_release_to_os_interval_ms); + max_user_defined_malloc_size = common_flags()->max_allocation_size_mb + ? common_flags()->max_allocation_size_mb + << 20 + : kMaxAllowedMallocSize; + } + + bool RssLimitExceeded() { + return atomic_load(&rss_limit_exceeded, memory_order_relaxed); + } + + void SetRssLimitExceeded(bool limit_exceeded) { + atomic_store(&rss_limit_exceeded, limit_exceeded, memory_order_relaxed); + } + + // -------------------- Allocation/Deallocation routines --------------- + void *Allocate(uptr size, uptr alignment, BufferedStackTrace *stack, + AllocType alloc_type) { + if (UNLIKELY(!heapprof_inited)) + HeapprofInitFromRtl(); + if (RssLimitExceeded()) { + if (AllocatorMayReturnNull()) + return nullptr; + ReportRssLimitExceeded(stack); + } + CHECK(stack); + const uptr min_alignment = HEAPPROF_ALIGNMENT; + if (alignment < min_alignment) + alignment = min_alignment; + if (size == 0) { + // We'd be happy to avoid allocating memory for zero-size requests, but + // some programs/tests depend on this behavior and assume that malloc + // would not return NULL even for zero-size allocations. Moreover, it + // looks like operator new should never return NULL, and results of + // consecutive "new" calls must be different even if the allocated size + // is zero. + size = 1; + } + CHECK(IsPowerOfTwo(alignment)); + uptr rounded_size = RoundUpTo(size, alignment); + uptr needed_size = rounded_size + kChunkHeaderSize; + if (alignment > min_alignment) + needed_size += alignment; + bool using_primary_allocator = true; + if (!PrimaryAllocator::CanAllocate(needed_size, alignment)) + using_primary_allocator = false; + CHECK(IsAligned(needed_size, min_alignment)); + if (size > kMaxAllowedMallocSize || needed_size > kMaxAllowedMallocSize || + size > max_user_defined_malloc_size) { + if (AllocatorMayReturnNull()) { + Report("WARNING: HeapProfiler failed to allocate 0x%zx bytes\n", + (void *)size); + return nullptr; + } + uptr malloc_limit = + Min(kMaxAllowedMallocSize, max_user_defined_malloc_size); + ReportAllocationSizeTooBig(size, needed_size, malloc_limit, stack); + } + + HeapprofThread *t = GetCurrentThread(); + void *allocated; + if (t) { + AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage()); + allocated = allocator.Allocate(cache, needed_size, 8); + } else { + SpinMutexLock l(&fallback_mutex); + AllocatorCache *cache = &fallback_allocator_cache; + allocated = allocator.Allocate(cache, needed_size, 8); + } + if (UNLIKELY(!allocated)) { + SetAllocatorOutOfMemory(); + if (AllocatorMayReturnNull()) + return nullptr; + ReportOutOfMemory(size, stack); + } + + uptr alloc_beg = reinterpret_cast(allocated); + uptr alloc_end = alloc_beg + needed_size; + uptr beg_plus_header = alloc_beg + kChunkHeaderSize; + uptr user_beg = beg_plus_header; + if (!IsAligned(user_beg, alignment)) + user_beg = RoundUpTo(user_beg, alignment); + uptr user_end = user_beg + size; + CHECK_LE(user_end, alloc_end); + uptr chunk_beg = user_beg - kChunkHeaderSize; + HeapprofChunk *m = reinterpret_cast(chunk_beg); + m->from_memalign = user_beg != beg_plus_header; + if (alloc_beg != chunk_beg) { + CHECK_LE(alloc_beg + 2 * sizeof(uptr), chunk_beg); + reinterpret_cast(alloc_beg)[0] = kAllocBegMagic; + reinterpret_cast(alloc_beg)[1] = chunk_beg; + } + if (using_primary_allocator) { + CHECK(size); + m->user_requested_size = size; + CHECK(allocator.FromPrimary(allocated)); + } else { + CHECK(!allocator.FromPrimary(allocated)); + m->user_requested_size = SizeClassMap::kMaxSize; + uptr *meta = reinterpret_cast(allocator.GetMetaData(allocated)); + meta[0] = size; + meta[1] = chunk_beg; + } + + m->cpu_id = GetCpuId(); + m->timestamp_ms = GetTimestamp(); + m->alloc_context_id = StackDepotPut(*stack); + + uptr size_rounded_down_to_granularity = + RoundDownTo(size, SHADOW_GRANULARITY); + if (size_rounded_down_to_granularity) + ClearShadow(user_beg, size_rounded_down_to_granularity); + + HeapprofStats &thread_stats = GetCurrentThreadStats(); + thread_stats.mallocs++; + thread_stats.malloced += size; + thread_stats.malloced_overhead += needed_size - size; + if (needed_size > SizeClassMap::kMaxSize) + thread_stats.malloc_large++; + else + thread_stats.malloced_by_size[SizeClassMap::ClassID(needed_size)]++; + + void *res = reinterpret_cast(user_beg); + HEAPPROF_MALLOC_HOOK(res, size); + return res; + } + + void Deallocate(void *ptr, uptr delete_size, uptr delete_alignment, + BufferedStackTrace *stack, AllocType alloc_type) { + uptr p = reinterpret_cast(ptr); + if (p == 0) + return; + + uptr chunk_beg = p - kChunkHeaderSize; + HeapprofChunk *m = reinterpret_cast(chunk_beg); + + if (heapprof_inited && heapprof_init_done && !destructing) { + u64 c = GetShadowCount(p, m->user_requested_size); + long curtime = GetTimestamp(); + + HeapInfoBlock newHIB(m->user_requested_size, c, m->timestamp_ms, curtime, + m->cpu_id, GetCpuId()); + { + SpinMutexLock l(&fallback_mutex); + HeapInfoBlockTable.insertOrMerge(m->alloc_context_id, newHIB); + } + } + + HeapprofStats &thread_stats = GetCurrentThreadStats(); + thread_stats.frees++; + thread_stats.freed += m->UsedSize(); + m->user_requested_size = 0; + + HeapprofThread *t = GetCurrentThread(); + if (t) { + AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage()); + allocator.Deallocate(cache, m->AllocBeg()); + } else { + SpinMutexLock l(&fallback_mutex); + AllocatorCache *cache = &fallback_allocator_cache; + allocator.Deallocate(cache, m->AllocBeg()); + } + + HEAPPROF_FREE_HOOK(ptr); + } + + void *Reallocate(void *old_ptr, uptr new_size, BufferedStackTrace *stack) { + CHECK(old_ptr && new_size); + uptr p = reinterpret_cast(old_ptr); + uptr chunk_beg = p - kChunkHeaderSize; + HeapprofChunk *m = reinterpret_cast(chunk_beg); + + HeapprofStats &thread_stats = GetCurrentThreadStats(); + thread_stats.reallocs++; + thread_stats.realloced += new_size; + + void *new_ptr = Allocate(new_size, 8, stack, FROM_MALLOC); + if (new_ptr) { + CHECK_NE(REAL(memcpy), nullptr); + uptr memcpy_size = Min(new_size, m->UsedSize()); + REAL(memcpy)(new_ptr, old_ptr, memcpy_size); + Deallocate(old_ptr, 0, 0, stack, FROM_MALLOC); + } + return new_ptr; + } + + void *Calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) { + if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { + if (AllocatorMayReturnNull()) + return nullptr; + ReportCallocOverflow(nmemb, size, stack); + } + void *ptr = Allocate(nmemb * size, 8, stack, FROM_MALLOC); + // If the memory comes from the secondary allocator no need to clear it + // as it comes directly from mmap. + if (ptr && allocator.FromPrimary(ptr)) + REAL(memset)(ptr, 0, nmemb * size); + return ptr; + } + + void CommitBack(HeapprofThreadLocalMallocStorage *ms, + BufferedStackTrace *stack) { + AllocatorCache *ac = GetAllocatorCache(ms); + allocator.SwallowCache(ac); + } + + // -------------------------- Chunk lookup ---------------------- + + // Assumes alloc_beg == allocator.GetBlockBegin(alloc_beg). + HeapprofChunk *GetHeapprofChunk(void *alloc_beg) { + if (!alloc_beg) + return nullptr; + if (!allocator.FromPrimary(alloc_beg)) { + uptr *meta = reinterpret_cast(allocator.GetMetaData(alloc_beg)); + HeapprofChunk *m = reinterpret_cast(meta[1]); + return m; + } + uptr *alloc_magic = reinterpret_cast(alloc_beg); + if (alloc_magic[0] == kAllocBegMagic) + return reinterpret_cast(alloc_magic[1]); + return reinterpret_cast(alloc_beg); + } + + HeapprofChunk *GetHeapprofChunkByAddr(uptr p) { + void *alloc_beg = allocator.GetBlockBegin(reinterpret_cast(p)); + return GetHeapprofChunk(alloc_beg); + } + + uptr AllocationSize(uptr p) { + HeapprofChunk *m = GetHeapprofChunkByAddr(p); + if (!m) + return 0; + if (!m->user_requested_size) + return 0; + if (m->Beg() != p) + return 0; + return m->UsedSize(); + } + + void Purge(BufferedStackTrace *stack) { allocator.ForceReleaseToOS(); } + + void PrintStats() { allocator.PrintStats(); } + + void ForceLock() { + allocator.ForceLock(); + fallback_mutex.Lock(); + } + + void ForceUnlock() { + fallback_mutex.Unlock(); + allocator.ForceUnlock(); + } +}; + +static Allocator instance(LINKER_INITIALIZED); + +static HeapprofAllocator &get_allocator() { return instance.allocator; } + +void InitializeAllocator() { instance.InitLinkerInitialized(); } + +void HeapprofThreadLocalMallocStorage::CommitBack() { + GET_STACK_TRACE_MALLOC; + instance.CommitBack(this, &stack); +} + +void PrintInternalAllocatorStats() { instance.PrintStats(); } + +void heapprof_free(void *ptr, BufferedStackTrace *stack, AllocType alloc_type) { + instance.Deallocate(ptr, 0, 0, stack, alloc_type); +} + +void heapprof_delete(void *ptr, uptr size, uptr alignment, + BufferedStackTrace *stack, AllocType alloc_type) { + instance.Deallocate(ptr, size, alignment, stack, alloc_type); +} + +void *heapprof_malloc(uptr size, BufferedStackTrace *stack) { + return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC)); +} + +void *heapprof_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) { + return SetErrnoOnNull(instance.Calloc(nmemb, size, stack)); +} + +void *heapprof_reallocarray(void *p, uptr nmemb, uptr size, + BufferedStackTrace *stack) { + if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { + errno = errno_ENOMEM; + if (AllocatorMayReturnNull()) + return nullptr; + ReportReallocArrayOverflow(nmemb, size, stack); + } + return heapprof_realloc(p, nmemb * size, stack); +} + +void *heapprof_realloc(void *p, uptr size, BufferedStackTrace *stack) { + if (!p) + return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC)); + if (size == 0) { + if (flags()->allocator_frees_and_returns_null_on_realloc_zero) { + instance.Deallocate(p, 0, 0, stack, FROM_MALLOC); + return nullptr; + } + // Allocate a size of 1 if we shouldn't free() on Realloc to 0 + size = 1; + } + return SetErrnoOnNull(instance.Reallocate(p, size, stack)); +} + +void *heapprof_valloc(uptr size, BufferedStackTrace *stack) { + return SetErrnoOnNull( + instance.Allocate(size, GetPageSizeCached(), stack, FROM_MALLOC)); +} + +void *heapprof_pvalloc(uptr size, BufferedStackTrace *stack) { + uptr PageSize = GetPageSizeCached(); + if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) { + errno = errno_ENOMEM; + if (AllocatorMayReturnNull()) + return nullptr; + ReportPvallocOverflow(size, stack); + } + // pvalloc(0) should allocate one page. + size = size ? RoundUpTo(size, PageSize) : PageSize; + return SetErrnoOnNull(instance.Allocate(size, PageSize, stack, FROM_MALLOC)); +} + +void *heapprof_memalign(uptr alignment, uptr size, BufferedStackTrace *stack, + AllocType alloc_type) { + if (UNLIKELY(!IsPowerOfTwo(alignment))) { + errno = errno_EINVAL; + if (AllocatorMayReturnNull()) + return nullptr; + ReportInvalidAllocationAlignment(alignment, stack); + } + return SetErrnoOnNull(instance.Allocate(size, alignment, stack, alloc_type)); +} + +void *heapprof_aligned_alloc(uptr alignment, uptr size, + BufferedStackTrace *stack) { + if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) { + errno = errno_EINVAL; + if (AllocatorMayReturnNull()) + return nullptr; + ReportInvalidAlignedAllocAlignment(size, alignment, stack); + } + return SetErrnoOnNull(instance.Allocate(size, alignment, stack, FROM_MALLOC)); +} + +int heapprof_posix_memalign(void **memptr, uptr alignment, uptr size, + BufferedStackTrace *stack) { + if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) { + if (AllocatorMayReturnNull()) + return errno_EINVAL; + ReportInvalidPosixMemalignAlignment(alignment, stack); + } + void *ptr = instance.Allocate(size, alignment, stack, FROM_MALLOC); + if (UNLIKELY(!ptr)) + // OOM error is already taken care of by Allocate. + return errno_ENOMEM; + CHECK(IsAligned((uptr)ptr, alignment)); + *memptr = ptr; + return 0; +} + +uptr heapprof_malloc_usable_size(const void *ptr, uptr pc, uptr bp) { + if (!ptr) + return 0; + uptr usable_size = instance.AllocationSize(reinterpret_cast(ptr)); + return usable_size; +} + +uptr heapprof_mz_size(const void *ptr) { + return instance.AllocationSize(reinterpret_cast(ptr)); +} + +void heapprof_mz_force_lock() { instance.ForceLock(); } + +void heapprof_mz_force_unlock() { instance.ForceUnlock(); } + +void HeapprofSoftRssLimitExceededCallback(bool limit_exceeded) { + instance.SetRssLimitExceeded(limit_exceeded); +} + +} // namespace __heapprof + +// ---------------------- Interface ---------------- {{{1 +using namespace __heapprof; + +#if !SANITIZER_SUPPORTS_WEAK_HOOKS +// Provide default (no-op) implementation of malloc hooks. +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_malloc_hook, void *ptr, + uptr size) { + (void)ptr; + (void)size; +} + +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_free_hook, void *ptr) { + (void)ptr; +} +#endif diff --git a/compiler-rt/lib/heapprof/heapprof_descriptions.h b/compiler-rt/lib/heapprof/heapprof_descriptions.h new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_descriptions.h @@ -0,0 +1,45 @@ +//===-- heapprof_descriptions.h ---------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// HeapProf-private header for heapprof_descriptions.cpp. +//===----------------------------------------------------------------------===// +#ifndef HEAPPROF_DESCRIPTIONS_H +#define HEAPPROF_DESCRIPTIONS_H + +#include "heapprof_allocator.h" +#include "heapprof_thread.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_report_decorator.h" + +namespace __heapprof { + +void DescribeThread(HeapprofThreadContext *context); +static inline void DescribeThread(HeapprofThread *t) { + if (t) + DescribeThread(t->context()); +} + +class HeapprofThreadIdAndName { +public: + explicit HeapprofThreadIdAndName(HeapprofThreadContext *t); + explicit HeapprofThreadIdAndName(u32 tid); + + // Contains "T%tid (%name)" or "T%tid" if the name is empty. + const char *c_str() const { return &name[0]; } + +private: + void Init(u32 tid, const char *tname); + + char name[128]; +}; + +} // namespace __heapprof + +#endif // HEAPPROF_DESCRIPTIONS_H diff --git a/compiler-rt/lib/heapprof/heapprof_descriptions.cpp b/compiler-rt/lib/heapprof/heapprof_descriptions.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_descriptions.cpp @@ -0,0 +1,71 @@ +//===-- heapprof_descriptions.cpp -------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// HeapProf functions for getting information about an address and/or printing +// it. +//===----------------------------------------------------------------------===// + +#include "heapprof_descriptions.h" +#include "heapprof_mapping.h" +#include "heapprof_report.h" +#include "heapprof_stack.h" +#include "sanitizer_common/sanitizer_stackdepot.h" + +namespace __heapprof { + +HeapprofThreadIdAndName::HeapprofThreadIdAndName(HeapprofThreadContext *t) { + Init(t->tid, t->name); +} + +HeapprofThreadIdAndName::HeapprofThreadIdAndName(u32 tid) { + if (tid == kInvalidTid) { + Init(tid, ""); + } else { + heapprofThreadRegistry().CheckLocked(); + HeapprofThreadContext *t = GetThreadContextByTidLocked(tid); + Init(tid, t->name); + } +} + +void HeapprofThreadIdAndName::Init(u32 tid, const char *tname) { + int len = internal_snprintf(name, sizeof(name), "T%d", tid); + CHECK(((unsigned int)len) < sizeof(name)); + if (tname[0] != '\0') + internal_snprintf(&name[len], sizeof(name) - len, " (%s)", tname); +} + +void DescribeThread(HeapprofThreadContext *context) { + CHECK(context); + heapprofThreadRegistry().CheckLocked(); + // No need to announce the main thread. + if (context->tid == 0 || context->announced) { + return; + } + context->announced = true; + InternalScopedString str(1024); + str.append("Thread %s", HeapprofThreadIdAndName(context).c_str()); + if (context->parent_tid == kInvalidTid) { + str.append(" created by unknown thread\n"); + Printf("%s", str.data()); + return; + } + str.append(" created by %s here:\n", + HeapprofThreadIdAndName(context->parent_tid).c_str()); + Printf("%s", str.data()); + StackDepotGet(context->stack_id).Print(); + // Recursively described parent thread if needed. + if (flags()->print_full_thread_history) { + HeapprofThreadContext *parent_context = + GetThreadContextByTidLocked(context->parent_tid); + DescribeThread(parent_context); + } +} + +} // namespace __heapprof diff --git a/compiler-rt/lib/heapprof/heapprof_errors.h b/compiler-rt/lib/heapprof/heapprof_errors.h new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_errors.h @@ -0,0 +1,209 @@ +//===-- heapprof_errors.h ---------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// HeapProf-private header for error structures. +//===----------------------------------------------------------------------===// +#ifndef HEAPPROF_ERRORS_H +#define HEAPPROF_ERRORS_H + +#include "heapprof_descriptions.h" +#include "sanitizer_common/sanitizer_common.h" + +namespace __heapprof { + +// (*) VS2013 does not implement unrestricted unions, so we need a trivial +// default constructor explicitly defined for each particular error. + +// None of the error classes own the stack traces mentioned in them. + +struct ErrorBase { + char descr[1024]; + u32 tid; + + ErrorBase() = default; // (*) + explicit ErrorBase(u32 tid_) : tid(tid_) {} + ErrorBase(u32 tid_, int initial_score, const char *reason) : tid(tid_) { + descr[0] = 0; + internal_strlcat(descr, reason, sizeof(descr)); + Printf("descr: %s %s\n", descr, reason); + } +}; + +struct ErrorCallocOverflow : ErrorBase { + const BufferedStackTrace *stack; + uptr count; + uptr size; + + ErrorCallocOverflow() = default; // (*) + ErrorCallocOverflow(u32 tid, BufferedStackTrace *stack_, uptr count_, + uptr size_) + : ErrorBase(tid, 10, "calloc-overflow"), stack(stack_), count(count_), + size(size_) {} + void Print(); +}; + +struct ErrorReallocArrayOverflow : ErrorBase { + const BufferedStackTrace *stack; + uptr count; + uptr size; + + ErrorReallocArrayOverflow() = default; // (*) + ErrorReallocArrayOverflow(u32 tid, BufferedStackTrace *stack_, uptr count_, + uptr size_) + : ErrorBase(tid, 10, "reallocarray-overflow"), stack(stack_), + count(count_), size(size_) {} + void Print(); +}; + +struct ErrorPvallocOverflow : ErrorBase { + const BufferedStackTrace *stack; + uptr size; + + ErrorPvallocOverflow() = default; // (*) + ErrorPvallocOverflow(u32 tid, BufferedStackTrace *stack_, uptr size_) + : ErrorBase(tid, 10, "pvalloc-overflow"), stack(stack_), size(size_) {} + void Print(); +}; + +struct ErrorInvalidAllocationAlignment : ErrorBase { + const BufferedStackTrace *stack; + uptr alignment; + + ErrorInvalidAllocationAlignment() = default; // (*) + ErrorInvalidAllocationAlignment(u32 tid, BufferedStackTrace *stack_, + uptr alignment_) + : ErrorBase(tid, 10, "invalid-allocation-alignment"), stack(stack_), + alignment(alignment_) {} + void Print(); +}; + +struct ErrorInvalidAlignedAllocAlignment : ErrorBase { + const BufferedStackTrace *stack; + uptr size; + uptr alignment; + + ErrorInvalidAlignedAllocAlignment() = default; // (*) + ErrorInvalidAlignedAllocAlignment(u32 tid, BufferedStackTrace *stack_, + uptr size_, uptr alignment_) + : ErrorBase(tid, 10, "invalid-aligned-alloc-alignment"), stack(stack_), + size(size_), alignment(alignment_) {} + void Print(); +}; + +struct ErrorInvalidPosixMemalignAlignment : ErrorBase { + const BufferedStackTrace *stack; + uptr alignment; + + ErrorInvalidPosixMemalignAlignment() = default; // (*) + ErrorInvalidPosixMemalignAlignment(u32 tid, BufferedStackTrace *stack_, + uptr alignment_) + : ErrorBase(tid, 10, "invalid-posix-memalign-alignment"), stack(stack_), + alignment(alignment_) {} + void Print(); +}; + +struct ErrorAllocationSizeTooBig : ErrorBase { + const BufferedStackTrace *stack; + uptr user_size; + uptr total_size; + uptr max_size; + + ErrorAllocationSizeTooBig() = default; // (*) + ErrorAllocationSizeTooBig(u32 tid, BufferedStackTrace *stack_, + uptr user_size_, uptr total_size_, uptr max_size_) + : ErrorBase(tid, 10, "allocation-size-too-big"), stack(stack_), + user_size(user_size_), total_size(total_size_), max_size(max_size_) {} + void Print(); +}; + +struct ErrorRssLimitExceeded : ErrorBase { + const BufferedStackTrace *stack; + + ErrorRssLimitExceeded() = default; // (*) + ErrorRssLimitExceeded(u32 tid, BufferedStackTrace *stack_) + : ErrorBase(tid, 10, "rss-limit-exceeded"), stack(stack_) {} + void Print(); +}; + +struct ErrorOutOfMemory : ErrorBase { + const BufferedStackTrace *stack; + uptr requested_size; + + ErrorOutOfMemory() = default; // (*) + ErrorOutOfMemory(u32 tid, BufferedStackTrace *stack_, uptr requested_size_) + : ErrorBase(tid, 10, "out-of-memory"), stack(stack_), + requested_size(requested_size_) {} + void Print(); +}; + +// clang-format off +#define HEAPPROF_FOR_EACH_ERROR_KIND(macro) \ + macro(CallocOverflow) \ + macro(ReallocArrayOverflow) \ + macro(PvallocOverflow) \ + macro(InvalidAllocationAlignment) \ + macro(InvalidAlignedAllocAlignment) \ + macro(InvalidPosixMemalignAlignment) \ + macro(AllocationSizeTooBig) \ + macro(RssLimitExceeded) \ + macro(OutOfMemory) \ +// clang-format on + +#define HEAPPROF_DEFINE_ERROR_KIND(name) kErrorKind##name, +#define HEAPPROF_ERROR_DESCRIPTION_MEMBER(name) Error##name name; +#define HEAPPROF_ERROR_DESCRIPTION_CONSTRUCTOR(name) \ + ErrorDescription(Error##name const &e) : kind(kErrorKind##name) { \ + internal_memcpy(&name, &e, sizeof(name)); \ + } +#define HEAPPROF_ERROR_DESCRIPTION_PRINT(name) \ + case kErrorKind##name: \ + return name.Print(); + +enum ErrorKind { + kErrorKindInvalid = 0, + HEAPPROF_FOR_EACH_ERROR_KIND(HEAPPROF_DEFINE_ERROR_KIND) +}; + +struct ErrorDescription { + ErrorKind kind; + // We're using a tagged union because it allows us to have a trivially + // copiable type and use the same structures as the public interface. + // + // We can add a wrapper around it to make it "more c++-like", but that would + // add a lot of code and the benefit wouldn't be that big. + union { + ErrorBase Base; + HEAPPROF_FOR_EACH_ERROR_KIND(HEAPPROF_ERROR_DESCRIPTION_MEMBER) + }; + + ErrorDescription() { internal_memset(this, 0, sizeof(*this)); } + explicit ErrorDescription(LinkerInitialized) {} + HEAPPROF_FOR_EACH_ERROR_KIND(HEAPPROF_ERROR_DESCRIPTION_CONSTRUCTOR) + + bool IsValid() { return kind != kErrorKindInvalid; } + void Print() { + switch (kind) { + HEAPPROF_FOR_EACH_ERROR_KIND(HEAPPROF_ERROR_DESCRIPTION_PRINT) + case kErrorKindInvalid: + CHECK(0); + } + CHECK(0); + } +}; + +#undef HEAPPROF_FOR_EACH_ERROR_KIND +#undef HEAPPROF_DEFINE_ERROR_KIND +#undef HEAPPROF_ERROR_DESCRIPTION_MEMBER +#undef HEAPPROF_ERROR_DESCRIPTION_CONSTRUCTOR +#undef HEAPPROF_ERROR_DESCRIPTION_PRINT + +} // namespace __heapprof + +#endif // HEAPPROF_ERRORS_H diff --git a/compiler-rt/lib/heapprof/heapprof_errors.cpp b/compiler-rt/lib/heapprof/heapprof_errors.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_errors.cpp @@ -0,0 +1,138 @@ +//===-- heapprof_errors.cpp -------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// HeapProf implementation for error structures. +//===----------------------------------------------------------------------===// + +#include "heapprof_errors.h" +#include "heapprof_descriptions.h" +#include "heapprof_mapping.h" +#include "heapprof_report.h" +#include "heapprof_stack.h" +#include "sanitizer_common/sanitizer_report_decorator.h" +#include "sanitizer_common/sanitizer_stackdepot.h" + +namespace __heapprof { + +void ErrorCallocOverflow::Print() { + SanitizerCommonDecorator d; + Printf("%s", d.Error()); + Report("ERROR: HeapProfiler: calloc parameters overflow: count * size " + "(%zd * %zd) cannot be represented in type size_t (thread %s)\n", + count, size, HeapprofThreadIdAndName(tid).c_str()); + Printf("%s", d.Default()); + stack->Print(); + PrintHintAllocatorCannotReturnNull(); + ReportErrorSummary(descr, stack); +} + +void ErrorReallocArrayOverflow::Print() { + SanitizerCommonDecorator d; + Printf("%s", d.Error()); + Report("ERROR: HeapProfiler: reallocarray parameters overflow: count * size " + "(%zd * %zd) cannot be represented in type size_t (thread %s)\n", + count, size, HeapprofThreadIdAndName(tid).c_str()); + Printf("%s", d.Default()); + stack->Print(); + PrintHintAllocatorCannotReturnNull(); + ReportErrorSummary(descr, stack); +} + +void ErrorPvallocOverflow::Print() { + SanitizerCommonDecorator d; + Printf("%s", d.Error()); + Report("ERROR: HeapProfiler: pvalloc parameters overflow: size 0x%zx " + "rounded up to system page size 0x%zx cannot be represented in type " + "size_t (thread %s)\n", + size, GetPageSizeCached(), HeapprofThreadIdAndName(tid).c_str()); + Printf("%s", d.Default()); + stack->Print(); + PrintHintAllocatorCannotReturnNull(); + ReportErrorSummary(descr, stack); +} + +void ErrorInvalidAllocationAlignment::Print() { + SanitizerCommonDecorator d; + Printf("%s", d.Error()); + Report("ERROR: HeapProfiler: invalid allocation alignment: %zd, " + "alignment must be a power of two (thread %s)\n", + alignment, HeapprofThreadIdAndName(tid).c_str()); + Printf("%s", d.Default()); + stack->Print(); + PrintHintAllocatorCannotReturnNull(); + ReportErrorSummary(descr, stack); +} + +void ErrorInvalidAlignedAllocAlignment::Print() { + SanitizerCommonDecorator d; + Printf("%s", d.Error()); + Report("ERROR: HeapProfiler: invalid alignment requested in " + "aligned_alloc: %zd, alignment must be a power of two and the " + "requested size 0x%zx must be a multiple of alignment " + "(thread %s)\n", + alignment, size, HeapprofThreadIdAndName(tid).c_str()); + Printf("%s", d.Default()); + stack->Print(); + PrintHintAllocatorCannotReturnNull(); + ReportErrorSummary(descr, stack); +} + +void ErrorInvalidPosixMemalignAlignment::Print() { + SanitizerCommonDecorator d; + Printf("%s", d.Error()); + Report( + "ERROR: HeapProfiler: invalid alignment requested in posix_memalign: " + "%zd, alignment must be a power of two and a multiple of sizeof(void*) " + "== %zd (thread %s)\n", + alignment, sizeof(void *), HeapprofThreadIdAndName(tid).c_str()); + Printf("%s", d.Default()); + stack->Print(); + PrintHintAllocatorCannotReturnNull(); + ReportErrorSummary(descr, stack); +} + +void ErrorAllocationSizeTooBig::Print() { + SanitizerCommonDecorator d; + Printf("%s", d.Error()); + Report("ERROR: HeapProfiler: requested allocation size 0x%zx (0x%zx after " + "adjustments for alignment, headers etc.) exceeds maximum supported " + "size of 0x%zx (thread %s)\n", + user_size, total_size, max_size, HeapprofThreadIdAndName(tid).c_str()); + Printf("%s", d.Default()); + stack->Print(); + PrintHintAllocatorCannotReturnNull(); + ReportErrorSummary(descr, stack); +} + +void ErrorRssLimitExceeded::Print() { + SanitizerCommonDecorator d; + Printf("%s", d.Error()); + Report("ERROR: HeapProfiler: specified RSS limit exceeded, currently set to " + "soft_rss_limit_mb=%zd\n", + common_flags()->soft_rss_limit_mb); + Printf("%s", d.Default()); + stack->Print(); + PrintHintAllocatorCannotReturnNull(); + ReportErrorSummary(descr, stack); +} + +void ErrorOutOfMemory::Print() { + SanitizerCommonDecorator d; + Printf("%s", d.Error()); + Report("ERROR: HeapProfiler: allocator is out of memory trying to allocate " + "0x%zx bytes\n", + requested_size); + Printf("%s", d.Default()); + stack->Print(); + PrintHintAllocatorCannotReturnNull(); + ReportErrorSummary(descr, stack); +} + +} // namespace __heapprof diff --git a/compiler-rt/lib/heapprof/heapprof_flags.h b/compiler-rt/lib/heapprof/heapprof_flags.h new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_flags.h @@ -0,0 +1,45 @@ +//===-- heapprof_flags.h ---------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// HeapProf runtime flags. +//===----------------------------------------------------------------------===// + +#ifndef HEAPPROF_FLAGS_H +#define HEAPPROF_FLAGS_H + +#include "sanitizer_common/sanitizer_flag_parser.h" +#include "sanitizer_common/sanitizer_internal_defs.h" + +// HeapProf flag values can be defined in four ways: +// 1) initialized with default values at startup. +// 2) overriden during compilation of HeapProf runtime by providing +// compile definition HEAPPROF_DEFAULT_OPTIONS. +// 3) overriden from string returned by user-specified function +// __heapprof_default_options(). +// 4) overriden from env variable HEAPPROF_OPTIONS. + +namespace __heapprof { + +struct Flags { +#define HEAPPROF_FLAG(Type, Name, DefaultValue, Description) Type Name; +#include "heapprof_flags.inc" +#undef HEAPPROF_FLAG + + void SetDefaults(); +}; + +extern Flags heapprof_flags_dont_use_directly; +inline Flags *flags() { return &heapprof_flags_dont_use_directly; } + +void InitializeFlags(); + +} // namespace __heapprof + +#endif // HEAPPROF_FLAGS_H diff --git a/compiler-rt/lib/heapprof/heapprof_flags.cpp b/compiler-rt/lib/heapprof/heapprof_flags.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_flags.cpp @@ -0,0 +1,111 @@ +//===-- heapprof_flags.cpp --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// HeapProf flag parsing logic. +//===----------------------------------------------------------------------===// + +#include "heapprof_flags.h" +#include "heapprof_interface_internal.h" +#include "heapprof_stack.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_flag_parser.h" +#include "sanitizer_common/sanitizer_flags.h" + +namespace __heapprof { + +Flags heapprof_flags_dont_use_directly; // use via flags(). + +static const char *MaybeCallHeapprofDefaultOptions() { + return (&__heapprof_default_options) ? __heapprof_default_options() : ""; +} + +static const char *MaybeUseHeapprofDefaultOptionsCompileDefinition() { +#ifdef HEAPPROF_DEFAULT_OPTIONS + return SANITIZER_STRINGIFY(HEAPPROF_DEFAULT_OPTIONS); +#else + return ""; +#endif +} + +void Flags::SetDefaults() { +#define HEAPPROF_FLAG(Type, Name, DefaultValue, Description) \ + Name = DefaultValue; +#include "heapprof_flags.inc" +#undef HEAPPROF_FLAG +} + +static void RegisterHeapprofFlags(FlagParser *parser, Flags *f) { +#define HEAPPROF_FLAG(Type, Name, DefaultValue, Description) \ + RegisterFlag(parser, #Name, Description, &f->Name); +#include "heapprof_flags.inc" +#undef HEAPPROF_FLAG +} + +void InitializeFlags() { + // Set the default values and prepare for parsing HeapProf and common flags. + SetCommonFlagsDefaults(); + { + CommonFlags cf; + cf.CopyFrom(*common_flags()); + cf.external_symbolizer_path = GetEnv("HEAPPROF_SYMBOLIZER_PATH"); + cf.malloc_context_size = kDefaultMallocContextSize; + cf.intercept_tls_get_addr = true; + cf.exitcode = 1; + OverrideCommonFlags(cf); + } + Flags *f = flags(); + f->SetDefaults(); + + FlagParser heapprof_parser; + RegisterHeapprofFlags(&heapprof_parser, f); + RegisterCommonFlags(&heapprof_parser); + + // Override from HeapProf compile definition. + const char *heapprof_compile_def = + MaybeUseHeapprofDefaultOptionsCompileDefinition(); + heapprof_parser.ParseString(heapprof_compile_def); + + // Override from user-specified string. + const char *heapprof_default_options = MaybeCallHeapprofDefaultOptions(); + heapprof_parser.ParseString(heapprof_default_options); + + // Override from command line. + heapprof_parser.ParseStringFromEnv("HEAPPROF_OPTIONS"); + + InitializeCommonFlags(); + + if (Verbosity()) + ReportUnrecognizedFlags(); + + if (common_flags()->help) { + heapprof_parser.PrintFlagDescriptions(); + } + + CHECK_LE((uptr)common_flags()->malloc_context_size, kStackTraceMax); + + if (!f->replace_str && common_flags()->intercept_strlen) { + Report("WARNING: strlen interceptor is enabled even though replace_str=0. " + "Use intercept_strlen=0 to disable it."); + } + if (!f->replace_str && common_flags()->intercept_strchr) { + Report("WARNING: strchr* interceptors are enabled even though " + "replace_str=0. Use intercept_strchr=0 to disable them."); + } + if (!f->replace_str && common_flags()->intercept_strndup) { + Report("WARNING: strndup* interceptors are enabled even though " + "replace_str=0. Use intercept_strndup=0 to disable them."); + } +} + +} // namespace __heapprof + +SANITIZER_INTERFACE_WEAK_DEF(const char *, __heapprof_default_options, void) { + return ""; +} diff --git a/compiler-rt/lib/heapprof/heapprof_flags.inc b/compiler-rt/lib/heapprof/heapprof_flags.inc new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_flags.inc @@ -0,0 +1,76 @@ +//===-- heapprof_flags.inc --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// HeapProf runtime flags. +// +//===----------------------------------------------------------------------===// +#ifndef HEAPPROF_FLAG +#error "Define HEAPPROF_FLAG prior to including this file!" +#endif + +// HEAPPROF_FLAG(Type, Name, DefaultValue, Description) +// See COMMON_FLAG in sanitizer_flags.inc for more details. + +HEAPPROF_FLAG( + bool, replace_str, true, + "If set, uses custom wrappers and replacements for libc string functions " + "to find more errors.") +HEAPPROF_FLAG( + bool, replace_intrin, true, + "If set, uses custom wrappers for memset/memcpy/memmove intrinsics.") +HEAPPROF_FLAG( + int, sleep_before_dying, 0, + "Number of seconds to sleep between printing an error report and " + "terminating the program. Useful for debugging purposes (e.g. when one " + "needs to attach gdb).") +HEAPPROF_FLAG( + int, sleep_after_init, 0, + "Number of seconds to sleep after HeapProfiler is initialized. " + "Useful for debugging purposes (e.g. when one needs to attach gdb).") +HEAPPROF_FLAG(bool, unmap_shadow_on_exit, false, + "If set, explicitly unmaps the (huge) shadow at exit.") +HEAPPROF_FLAG(bool, protect_shadow_gap, true, "If set, mprotect the shadow gap") +HEAPPROF_FLAG(bool, print_stats, false, + "Print various statistics after printing an error message or if " + "atexit=1.") +HEAPPROF_FLAG(bool, print_legend, true, + "Print the legend for the shadow bytes.") +HEAPPROF_FLAG( + bool, atexit, false, + "If set, prints HeapProf exit stats even after program terminates " + "successfully.") +HEAPPROF_FLAG( + bool, print_full_thread_history, true, + "If set, prints thread creation stacks for the threads involved in the " + "report and their ancestors up to the main thread.") + +HEAPPROF_FLAG(bool, halt_on_error, true, + "Crash the program after printing the first error report " + "(WARNING: USE AT YOUR OWN RISK!)") +HEAPPROF_FLAG(bool, allocator_frees_and_returns_null_on_realloc_zero, true, + "realloc(p, 0) is equivalent to free(p) by default (Same as the " + "POSIX standard). If set to false, realloc(p, 0) will return a " + "pointer to an allocated space which can not be used.") +HEAPPROF_FLAG( + bool, verify_heapprof_link_order, true, + "Check position of HeapProf runtime in library list (needs to be disabled" + " when other library has to be preloaded system-wide)") +HEAPPROF_FLAG(bool, print_terse, false, + "If set, prints heap profile in a terse format.") +HEAPPROF_FLAG(bool, dump_process_map, false, + "If set, prints the process memory map.") + +HEAPPROF_FLAG( + int, heap_info_cache_entries, 16381, + "Size in entries of the heap info block cache, should be closest prime" + " number to a power of two for best hashing.") +HEAPPROF_FLAG(bool, print_heap_info_cache_miss_rate, false, + "If set, prints the miss rate of the heap info block cache.") +HEAPPROF_FLAG( + bool, print_heap_info_cache_miss_rate_details, false, + "If set, prints detailed miss rates of the heap info block cache sets.") diff --git a/compiler-rt/lib/heapprof/heapprof_init_version.h b/compiler-rt/lib/heapprof/heapprof_init_version.h new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_init_version.h @@ -0,0 +1,26 @@ +//===-- heapprof_init_version.h ---------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// This header defines a versioned __heapprof_init function to be called at the +// startup of the instrumented program. +//===----------------------------------------------------------------------===// +#ifndef HEAPPROF_INIT_VERSION_H +#define HEAPPROF_INIT_VERSION_H + +#include "sanitizer_common/sanitizer_platform.h" + +extern "C" { +// Every time the Heapprof ABI changes we also change the version number in the +// __heapprof_init function name. Objects built with incompatible Heapprof ABI +// versions will not link with run-time. +#define __heapprof_version_mismatch_check __heapprof_version_mismatch_check_v1 +} + +#endif // HEAPPROF_INIT_VERSION_H diff --git a/compiler-rt/lib/heapprof/heapprof_interceptors.h b/compiler-rt/lib/heapprof/heapprof_interceptors.h new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_interceptors.h @@ -0,0 +1,54 @@ +//===-- heapprof_interceptors.h ---------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// HeapProf-private header for heapprof_interceptors.cpp +//===----------------------------------------------------------------------===// +#ifndef HEAPPROF_INTERCEPTORS_H +#define HEAPPROF_INTERCEPTORS_H + +#include "heapprof_interceptors_memintrinsics.h" +#include "heapprof_internal.h" +#include "interception/interception.h" +#include "sanitizer_common/sanitizer_platform_interceptors.h" + +namespace __heapprof { + +void InitializeHeapprofInterceptors(); +void InitializePlatformInterceptors(); + +#define ENSURE_HEAPPROF_INITED() \ + do { \ + CHECK(!heapprof_init_is_running); \ + if (UNLIKELY(!heapprof_inited)) { \ + HeapprofInitFromRtl(); \ + } \ + } while (0) + +} // namespace __heapprof + +DECLARE_REAL(int, memcmp, const void *a1, const void *a2, uptr size) +DECLARE_REAL(char *, strchr, const char *str, int c) +DECLARE_REAL(SIZE_T, strlen, const char *s) +DECLARE_REAL(char *, strncpy, char *to, const char *from, uptr size) +DECLARE_REAL(uptr, strnlen, const char *s, uptr maxlen) +DECLARE_REAL(char *, strstr, const char *s1, const char *s2) + +#define HEAPPROF_INTERCEPT_FUNC(name) \ + do { \ + if (!INTERCEPT_FUNCTION(name)) \ + VReport(1, "HeapProfiler: failed to intercept '%s'\n'", #name); \ + } while (0) +#define HEAPPROF_INTERCEPT_FUNC_VER(name, ver) \ + do { \ + if (!INTERCEPT_FUNCTION_VER(name, ver)) \ + VReport(1, "HeapProfiler: failed to intercept '%s@@%s'\n", #name, #ver); \ + } while (0) + +#endif // HEAPPROF_INTERCEPTORS_H diff --git a/compiler-rt/lib/heapprof/heapprof_interceptors.cpp b/compiler-rt/lib/heapprof/heapprof_interceptors.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_interceptors.cpp @@ -0,0 +1,399 @@ +//===-- heapprof_interceptors.cpp -----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// Intercept various libc functions. +//===----------------------------------------------------------------------===// + +#include "heapprof_interceptors.h" +#include "heapprof_allocator.h" +#include "heapprof_internal.h" +#include "heapprof_mapping.h" +#include "heapprof_report.h" +#include "heapprof_stack.h" +#include "heapprof_stats.h" +#include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_posix.h" + +namespace __heapprof { + +#define HEAPPROF_READ_STRING(s, n) HEAPPROF_READ_RANGE((s), (n)) + +static inline uptr MaybeRealStrnlen(const char *s, uptr maxlen) { +#if SANITIZER_INTERCEPT_STRNLEN + if (REAL(strnlen)) { + return REAL(strnlen)(s, maxlen); + } +#endif + return internal_strnlen(s, maxlen); +} + +void SetThreadName(const char *name) { + HeapprofThread *t = GetCurrentThread(); + if (t) + heapprofThreadRegistry().SetThreadName(t->tid(), name); +} + +int OnExit() { + // FIXME: ask frontend whether we need to return failure. + return 0; +} + +} // namespace __heapprof + +// ---------------------- Wrappers ---------------- {{{1 +using namespace __heapprof; + +DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr) +DECLARE_REAL_AND_INTERCEPTOR(void, free, void *) + +#define HEAPPROF_INTERCEPTOR_ENTER(ctx, func) \ + ctx = 0; \ + (void)ctx; + +#define COMMON_INTERCEPT_FUNCTION(name) HEAPPROF_INTERCEPT_FUNC(name) +#define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \ + HEAPPROF_INTERCEPT_FUNC_VER(name, ver) +#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \ + HEAPPROF_WRITE_RANGE(ptr, size) +#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \ + HEAPPROF_READ_RANGE(ptr, size) +#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \ + HEAPPROF_INTERCEPTOR_ENTER(ctx, func); \ + do { \ + if (heapprof_init_is_running) \ + return REAL(func)(__VA_ARGS__); \ + ENSURE_HEAPPROF_INITED(); \ + } while (false) +#define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \ + do { \ + } while (false) +#define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \ + do { \ + } while (false) +#define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \ + do { \ + } while (false) +#define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \ + do { \ + } while (false) +#define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) SetThreadName(name) +// Should be heapprofThreadRegistry().SetThreadNameByUserId(thread, name) +// But heapprof does not remember UserId's for threads (pthread_t); +// and remembers all ever existed threads, so the linear search by UserId +// can be slow. +#define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \ + do { \ + } while (false) +#define COMMON_INTERCEPTOR_BLOCK_REAL(name) REAL(name) +#define COMMON_INTERCEPTOR_ON_DLOPEN(filename, flag) \ + do { \ + CheckNoDeepBind(filename, flag); \ + } while (false) +#define COMMON_INTERCEPTOR_ON_EXIT(ctx) OnExit() +#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) +#define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() +#define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED (!heapprof_inited) +#define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end) \ + if (HeapprofThread *t = GetCurrentThread()) { \ + *begin = t->tls_begin(); \ + *end = t->tls_end(); \ + } else { \ + *begin = *end = 0; \ + } + +#define COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size) \ + do { \ + HEAPPROF_INTERCEPTOR_ENTER(ctx, memmove); \ + HEAPPROF_MEMMOVE_IMPL(to, from, size); \ + } while (false) + +#define COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size) \ + do { \ + HEAPPROF_INTERCEPTOR_ENTER(ctx, memcpy); \ + HEAPPROF_MEMCPY_IMPL(to, from, size); \ + } while (false) + +#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size) \ + do { \ + HEAPPROF_INTERCEPTOR_ENTER(ctx, memset); \ + HEAPPROF_MEMSET_IMPL(block, c, size); \ + } while (false) + +#include "sanitizer_common/sanitizer_common_interceptors.inc" +#include "sanitizer_common/sanitizer_signal_interceptors.inc" + +#define COMMON_SYSCALL_PRE_READ_RANGE(p, s) HEAPPROF_READ_RANGE(p, s) +#define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) HEAPPROF_WRITE_RANGE(p, s) +#define COMMON_SYSCALL_POST_READ_RANGE(p, s) \ + do { \ + (void)(p); \ + (void)(s); \ + } while (false) +#define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \ + do { \ + (void)(p); \ + (void)(s); \ + } while (false) +#include "sanitizer_common/sanitizer_common_syscalls.inc" + +struct ThreadStartParam { + atomic_uintptr_t t; + atomic_uintptr_t is_registered; +}; + +static thread_return_t THREAD_CALLING_CONV heapprof_thread_start(void *arg) { + ThreadStartParam *param = reinterpret_cast(arg); + HeapprofThread *t = nullptr; + while ((t = reinterpret_cast( + atomic_load(¶m->t, memory_order_acquire))) == nullptr) + internal_sched_yield(); + SetCurrentThread(t); + return t->ThreadStart(GetTid(), ¶m->is_registered); +} + +INTERCEPTOR(int, pthread_create, void *thread, void *attr, + void *(*start_routine)(void *), void *arg) { + EnsureMainThreadIDIsCorrect(); + GET_STACK_TRACE_THREAD; + int detached = 0; + if (attr) + REAL(pthread_attr_getdetachstate)(attr, &detached); + ThreadStartParam param; + atomic_store(¶m.t, 0, memory_order_relaxed); + atomic_store(¶m.is_registered, 0, memory_order_relaxed); + int result; + { + // Ignore all allocations made by pthread_create: thread stack/TLS may be + // stored by pthread for future reuse even after thread destruction, and + // the linked list it's stored in doesn't even hold valid pointers to the + // objects, the latter are calculated by obscure pointer arithmetic. + result = REAL(pthread_create)(thread, attr, heapprof_thread_start, ¶m); + } + if (result == 0) { + u32 current_tid = GetCurrentTidOrInvalid(); + HeapprofThread *t = HeapprofThread::Create(start_routine, arg, current_tid, + &stack, detached); + atomic_store(¶m.t, reinterpret_cast(t), memory_order_release); + // Wait until the HeapprofThread object is initialized and the + // ThreadRegistry entry is in "started" state. + while (atomic_load(¶m.is_registered, memory_order_acquire) == 0) + internal_sched_yield(); + } + return result; +} + +INTERCEPTOR(int, pthread_join, void *t, void **arg) { + return real_pthread_join(t, arg); +} + +DEFINE_REAL_PTHREAD_FUNCTIONS + +INTERCEPTOR(char *, index, const char *string, int c) +ALIAS(WRAPPER_NAME(strchr)); + +// For both strcat() and strncat() we need to check the validity of |to| +// argument irrespective of the |from| length. +INTERCEPTOR(char *, strcat, char *to, const char *from) { + void *ctx; + HEAPPROF_INTERCEPTOR_ENTER(ctx, strcat); + ENSURE_HEAPPROF_INITED(); + if (flags()->replace_str) { + uptr from_length = REAL(strlen)(from); + HEAPPROF_READ_RANGE(from, from_length + 1); + uptr to_length = REAL(strlen)(to); + HEAPPROF_READ_STRING(to, to_length); + HEAPPROF_WRITE_RANGE(to + to_length, from_length + 1); + } + return REAL(strcat)(to, from); +} + +INTERCEPTOR(char *, strncat, char *to, const char *from, uptr size) { + void *ctx; + HEAPPROF_INTERCEPTOR_ENTER(ctx, strncat); + ENSURE_HEAPPROF_INITED(); + if (flags()->replace_str) { + uptr from_length = MaybeRealStrnlen(from, size); + uptr copy_length = Min(size, from_length + 1); + HEAPPROF_READ_RANGE(from, copy_length); + uptr to_length = REAL(strlen)(to); + HEAPPROF_READ_STRING(to, to_length); + HEAPPROF_WRITE_RANGE(to + to_length, from_length + 1); + } + return REAL(strncat)(to, from, size); +} + +INTERCEPTOR(char *, strcpy, char *to, const char *from) { + void *ctx; + HEAPPROF_INTERCEPTOR_ENTER(ctx, strcpy); + if (heapprof_init_is_running) { + return REAL(strcpy)(to, from); + } + ENSURE_HEAPPROF_INITED(); + if (flags()->replace_str) { + uptr from_size = REAL(strlen)(from) + 1; + HEAPPROF_READ_RANGE(from, from_size); + HEAPPROF_WRITE_RANGE(to, from_size); + } + return REAL(strcpy)(to, from); +} + +INTERCEPTOR(char *, strdup, const char *s) { + void *ctx; + HEAPPROF_INTERCEPTOR_ENTER(ctx, strdup); + if (UNLIKELY(!heapprof_inited)) + return internal_strdup(s); + ENSURE_HEAPPROF_INITED(); + uptr length = REAL(strlen)(s); + if (flags()->replace_str) { + HEAPPROF_READ_RANGE(s, length + 1); + } + GET_STACK_TRACE_MALLOC; + void *new_mem = heapprof_malloc(length + 1, &stack); + REAL(memcpy)(new_mem, s, length + 1); + return reinterpret_cast(new_mem); +} + +INTERCEPTOR(char *, __strdup, const char *s) { + void *ctx; + HEAPPROF_INTERCEPTOR_ENTER(ctx, strdup); + if (UNLIKELY(!heapprof_inited)) + return internal_strdup(s); + ENSURE_HEAPPROF_INITED(); + uptr length = REAL(strlen)(s); + if (flags()->replace_str) { + HEAPPROF_READ_RANGE(s, length + 1); + } + GET_STACK_TRACE_MALLOC; + void *new_mem = heapprof_malloc(length + 1, &stack); + REAL(memcpy)(new_mem, s, length + 1); + return reinterpret_cast(new_mem); +} + +INTERCEPTOR(char *, strncpy, char *to, const char *from, uptr size) { + void *ctx; + HEAPPROF_INTERCEPTOR_ENTER(ctx, strncpy); + ENSURE_HEAPPROF_INITED(); + if (flags()->replace_str) { + uptr from_size = Min(size, MaybeRealStrnlen(from, size) + 1); + HEAPPROF_READ_RANGE(from, from_size); + HEAPPROF_WRITE_RANGE(to, size); + } + return REAL(strncpy)(to, from, size); +} + +INTERCEPTOR(long, strtol, const char *nptr, char **endptr, int base) { + void *ctx; + HEAPPROF_INTERCEPTOR_ENTER(ctx, strtol); + ENSURE_HEAPPROF_INITED(); + if (!flags()->replace_str) { + return REAL(strtol)(nptr, endptr, base); + } + char *real_endptr; + long result = REAL(strtol)(nptr, &real_endptr, base); + StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base); + return result; +} + +INTERCEPTOR(int, atoi, const char *nptr) { + void *ctx; + HEAPPROF_INTERCEPTOR_ENTER(ctx, atoi); + ENSURE_HEAPPROF_INITED(); + if (!flags()->replace_str) { + return REAL(atoi)(nptr); + } + char *real_endptr; + // "man atoi" tells that behavior of atoi(nptr) is the same as + // strtol(nptr, 0, 10), i.e. it sets errno to ERANGE if the + // parsed integer can't be stored in *long* type (even if it's + // different from int). So, we just imitate this behavior. + int result = REAL(strtol)(nptr, &real_endptr, 10); + FixRealStrtolEndptr(nptr, &real_endptr); + HEAPPROF_READ_STRING(nptr, (real_endptr - nptr) + 1); + return result; +} + +INTERCEPTOR(long, atol, const char *nptr) { + void *ctx; + HEAPPROF_INTERCEPTOR_ENTER(ctx, atol); + ENSURE_HEAPPROF_INITED(); + if (!flags()->replace_str) { + return REAL(atol)(nptr); + } + char *real_endptr; + long result = REAL(strtol)(nptr, &real_endptr, 10); + FixRealStrtolEndptr(nptr, &real_endptr); + HEAPPROF_READ_STRING(nptr, (real_endptr - nptr) + 1); + return result; +} + +INTERCEPTOR(long long, strtoll, const char *nptr, char **endptr, int base) { + void *ctx; + HEAPPROF_INTERCEPTOR_ENTER(ctx, strtoll); + ENSURE_HEAPPROF_INITED(); + if (!flags()->replace_str) { + return REAL(strtoll)(nptr, endptr, base); + } + char *real_endptr; + long long result = REAL(strtoll)(nptr, &real_endptr, base); + StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base); + return result; +} + +INTERCEPTOR(long long, atoll, const char *nptr) { + void *ctx; + HEAPPROF_INTERCEPTOR_ENTER(ctx, atoll); + ENSURE_HEAPPROF_INITED(); + if (!flags()->replace_str) { + return REAL(atoll)(nptr); + } + char *real_endptr; + long long result = REAL(strtoll)(nptr, &real_endptr, 10); + FixRealStrtolEndptr(nptr, &real_endptr); + HEAPPROF_READ_STRING(nptr, (real_endptr - nptr) + 1); + return result; +} + +// ---------------------- InitializeHeapprofInterceptors ---------------- {{{1 +namespace __heapprof { +void InitializeHeapprofInterceptors() { + static bool was_called_once; + CHECK(!was_called_once); + was_called_once = true; + InitializeCommonInterceptors(); + InitializeSignalInterceptors(); + + // Intercept str* functions. + HEAPPROF_INTERCEPT_FUNC(strcat); + HEAPPROF_INTERCEPT_FUNC(strcpy); + HEAPPROF_INTERCEPT_FUNC(strncat); + HEAPPROF_INTERCEPT_FUNC(strncpy); + HEAPPROF_INTERCEPT_FUNC(strdup); + HEAPPROF_INTERCEPT_FUNC(__strdup); + HEAPPROF_INTERCEPT_FUNC(index); + + HEAPPROF_INTERCEPT_FUNC(atoi); + HEAPPROF_INTERCEPT_FUNC(atol); + HEAPPROF_INTERCEPT_FUNC(strtol); + HEAPPROF_INTERCEPT_FUNC(atoll); + HEAPPROF_INTERCEPT_FUNC(strtoll); + + // Intercept threading-related functions +#if defined(HEAPPROF_PTHREAD_CREATE_VERSION) + HEAPPROF_INTERCEPT_FUNC_VER(pthread_create, HEAPPROF_PTHREAD_CREATE_VERSION); +#else + HEAPPROF_INTERCEPT_FUNC(pthread_create); +#endif + + InitializePlatformInterceptors(); + + VReport(1, "HeapProfiler: libc interceptors initialized\n"); +} + +} // namespace __heapprof diff --git a/compiler-rt/lib/heapprof/heapprof_interceptors_memintrinsics.h b/compiler-rt/lib/heapprof/heapprof_interceptors_memintrinsics.h new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_interceptors_memintrinsics.h @@ -0,0 +1,85 @@ +//===-- heapprof_interceptors_memintrinsics.h -------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===---------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// HeapProf-private header for heapprof_interceptors_memintrinsics.cpp +//===---------------------------------------------------------------------===// +#ifndef HEAPPROF_MEMINTRIN_H +#define HEAPPROF_MEMINTRIN_H + +#include "heapprof_interface_internal.h" +#include "heapprof_internal.h" +#include "heapprof_mapping.h" +#include "interception/interception.h" + +DECLARE_REAL(void *, memcpy, void *to, const void *from, uptr size) +DECLARE_REAL(void *, memset, void *block, int c, uptr size) + +namespace __heapprof { + +// We implement ACCESS_MEMORY_RANGE, HEAPPROF_READ_RANGE, +// and HEAPPROF_WRITE_RANGE as macro instead of function so +// that no extra frames are created, and stack trace contains +// relevant information only. +#define ACCESS_MEMORY_RANGE(offset, size) \ + do { \ + __heapprof_record_access_range(offset, size); \ + } while (0) + +// memcpy is called during __heapprof_init() from the internals of printf(...). +// We do not treat memcpy with to==from as a bug. +// See http://llvm.org/bugs/show_bug.cgi?id=11763. +#define HEAPPROF_MEMCPY_IMPL(to, from, size) \ + do { \ + if (UNLIKELY(!heapprof_inited)) \ + return internal_memcpy(to, from, size); \ + if (heapprof_init_is_running) { \ + return REAL(memcpy)(to, from, size); \ + } \ + ENSURE_HEAPPROF_INITED(); \ + if (flags()->replace_intrin) { \ + HEAPPROF_READ_RANGE(from, size); \ + HEAPPROF_WRITE_RANGE(to, size); \ + } \ + return REAL(memcpy)(to, from, size); \ + } while (0) + +// memset is called inside Printf. +#define HEAPPROF_MEMSET_IMPL(block, c, size) \ + do { \ + if (UNLIKELY(!heapprof_inited)) \ + return internal_memset(block, c, size); \ + if (heapprof_init_is_running) { \ + return REAL(memset)(block, c, size); \ + } \ + ENSURE_HEAPPROF_INITED(); \ + if (flags()->replace_intrin) { \ + HEAPPROF_WRITE_RANGE(block, size); \ + } \ + return REAL(memset)(block, c, size); \ + } while (0) + +#define HEAPPROF_MEMMOVE_IMPL(to, from, size) \ + do { \ + if (UNLIKELY(!heapprof_inited)) \ + return internal_memmove(to, from, size); \ + ENSURE_HEAPPROF_INITED(); \ + if (flags()->replace_intrin) { \ + HEAPPROF_READ_RANGE(from, size); \ + HEAPPROF_WRITE_RANGE(to, size); \ + } \ + return internal_memmove(to, from, size); \ + } while (0) + +#define HEAPPROF_READ_RANGE(offset, size) ACCESS_MEMORY_RANGE(offset, size) +#define HEAPPROF_WRITE_RANGE(offset, size) ACCESS_MEMORY_RANGE(offset, size) + +} // namespace __heapprof + +#endif // HEAPPROF_MEMINTRIN_H diff --git a/compiler-rt/lib/heapprof/heapprof_interceptors_memintrinsics.cpp b/compiler-rt/lib/heapprof/heapprof_interceptors_memintrinsics.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_interceptors_memintrinsics.cpp @@ -0,0 +1,30 @@ +//===-- heapprof_interceptors_memintrinsics.cpp ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===---------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// HeapProf versions of memcpy, memmove, and memset. +//===---------------------------------------------------------------------===// + +#include "heapprof_interceptors_memintrinsics.h" +#include "heapprof_report.h" +#include "heapprof_stack.h" + +using namespace __heapprof; + +void *__heapprof_memcpy(void *to, const void *from, uptr size) { + HEAPPROF_MEMCPY_IMPL(to, from, size); +} + +void *__heapprof_memset(void *block, int c, uptr size) { + HEAPPROF_MEMSET_IMPL(block, c, size); +} + +void *__heapprof_memmove(void *to, const void *from, uptr size) { + HEAPPROF_MEMMOVE_IMPL(to, from, size); +} diff --git a/compiler-rt/lib/heapprof/heapprof_interface.inc b/compiler-rt/lib/heapprof/heapprof_interface.inc new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_interface.inc @@ -0,0 +1,31 @@ +//===-- heapprof_interface.inc --------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Heapprof interface list. +//===----------------------------------------------------------------------===// +INTERFACE_FUNCTION(__heapprof_record_access) +INTERFACE_FUNCTION(__heapprof_record_access_range) +INTERFACE_FUNCTION(__heapprof_init) +INTERFACE_FUNCTION(__heapprof_preinit) +INTERFACE_FUNCTION(__heapprof_load) +INTERFACE_FUNCTION(__heapprof_memcpy) +INTERFACE_FUNCTION(__heapprof_memmove) +INTERFACE_FUNCTION(__heapprof_memset) +INTERFACE_FUNCTION(__heapprof_print_accumulated_stats) +INTERFACE_FUNCTION(__heapprof_store) +INTERFACE_FUNCTION(__heapprof_version_mismatch_check_v1) +INTERFACE_FUNCTION(__sanitizer_finish_switch_fiber) +INTERFACE_FUNCTION(__sanitizer_print_stack_trace) +INTERFACE_FUNCTION(__sanitizer_start_switch_fiber) +INTERFACE_FUNCTION(__sanitizer_unaligned_load16) +INTERFACE_FUNCTION(__sanitizer_unaligned_load32) +INTERFACE_FUNCTION(__sanitizer_unaligned_load64) +INTERFACE_FUNCTION(__sanitizer_unaligned_store16) +INTERFACE_FUNCTION(__sanitizer_unaligned_store32) +INTERFACE_FUNCTION(__sanitizer_unaligned_store64) +INTERFACE_WEAK_FUNCTION(__heapprof_default_options) +INTERFACE_WEAK_FUNCTION(__heapprof_on_error) diff --git a/compiler-rt/lib/heapprof/heapprof_interface_internal.h b/compiler-rt/lib/heapprof/heapprof_interface_internal.h new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_interface_internal.h @@ -0,0 +1,63 @@ +//===-- heapprof_interface_internal.h ---------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// This header declares the HeapProfiler runtime interface functions. +// The runtime library has to define these functions so the instrumented program +// could call them. +// +// See also include/sanitizer/heapprof_interface.h +//===----------------------------------------------------------------------===// +#ifndef HEAPPROF_INTERFACE_INTERNAL_H +#define HEAPPROF_INTERFACE_INTERNAL_H + +#include "sanitizer_common/sanitizer_internal_defs.h" + +#include "heapprof_init_version.h" + +using __sanitizer::u32; +using __sanitizer::u64; +using __sanitizer::uptr; + +extern "C" { +// This function should be called at the very beginning of the process, +// before any instrumented code is executed and before any call to malloc. +SANITIZER_INTERFACE_ATTRIBUTE void __heapprof_init(); +SANITIZER_INTERFACE_ATTRIBUTE void __heapprof_preinit(); +SANITIZER_INTERFACE_ATTRIBUTE void __heapprof_version_mismatch_check_v1(); + +SANITIZER_INTERFACE_ATTRIBUTE +void __heapprof_record_access(void const volatile *addr); + +SANITIZER_INTERFACE_ATTRIBUTE +void __heapprof_record_access_range(void const volatile *addr, uptr size); + +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void +__heapprof_on_error(); + +SANITIZER_INTERFACE_ATTRIBUTE void __heapprof_print_accumulated_stats(); + +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE const char * +__heapprof_default_options(); + +SANITIZER_INTERFACE_ATTRIBUTE +extern uptr __heapprof_shadow_memory_dynamic_address; + +SANITIZER_INTERFACE_ATTRIBUTE void __heapprof_load(uptr p); +SANITIZER_INTERFACE_ATTRIBUTE void __heapprof_store(uptr p); + +SANITIZER_INTERFACE_ATTRIBUTE +void *__heapprof_memcpy(void *dst, const void *src, uptr size); +SANITIZER_INTERFACE_ATTRIBUTE +void *__heapprof_memset(void *s, int c, uptr n); +SANITIZER_INTERFACE_ATTRIBUTE +void *__heapprof_memmove(void *dest, const void *src, uptr n); +} // extern "C" + +#endif // HEAPPROF_INTERFACE_INTERNAL_H diff --git a/compiler-rt/lib/heapprof/heapprof_internal.h b/compiler-rt/lib/heapprof/heapprof_internal.h new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_internal.h @@ -0,0 +1,118 @@ +//===-- heapprof_internal.h -------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// HeapProf-private header which defines various general utilities. +//===----------------------------------------------------------------------===// +#ifndef HEAPPROF_INTERNAL_H +#define HEAPPROF_INTERNAL_H + +#include "heapprof_flags.h" +#include "heapprof_interface_internal.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_internal_defs.h" +#include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_stacktrace.h" + +#if __has_feature(address_sanitizer) || defined(__SANITIZE_ADDRESS__) +#error "The HeapProfiler run-time should not be" +" instrumented by HeapProfiler" +#endif + +// Build-time configuration options. + +// If set, heapprof will intercept C++ exception api call(s). +#ifndef HEAPPROF_HAS_EXCEPTIONS +#define HEAPPROF_HAS_EXCEPTIONS 1 +#endif + +#ifndef HEAPPROF_DYNAMIC +#ifdef PIC +#define HEAPPROF_DYNAMIC 1 +#else +#define HEAPPROF_DYNAMIC 0 +#endif +#endif + + // All internal functions in heapprof reside inside the __heapprof namespace + // to avoid namespace collisions with the user programs. + // Separate namespace also makes it simpler to distinguish the heapprof + // run-time functions from the instrumented user code in a profile. + namespace __heapprof { + + class HeapprofThread; + using __sanitizer::StackTrace; + + void HeapprofInitFromRtl(); + + // heapprof_rtl.cpp + void PrintAddressSpaceLayout(); + void NORETURN ShowStatsAndAbort(); + + // heapprof_shadow_setup.cpp + void InitializeShadowMemory(); + + // heapprof_malloc_linux.cpp + void ReplaceSystemMalloc(); + + // heapprof_linux.cpp + uptr FindDynamicShadowStart(); + void *HeapprofDoesNotSupportStaticLinkage(); + void HeapprofCheckDynamicRTPrereqs(); + void HeapprofCheckIncompatibleRT(); + + // heapprof_thread.cpp + HeapprofThread *CreateMainThread(); + + void ReadContextStack(void *context, uptr *stack, uptr *ssize); + + // Wrapper for TLS/TSD. + void HeapprofTSDInit(void (*destructor)(void *tsd)); + void *HeapprofTSDGet(); + void HeapprofTSDSet(void *tsd); + void PlatformTSDDtor(void *tsd); + + void AppendToErrorMessageBuffer(const char *buffer); + + void *HeapprofDlSymNext(const char *sym); + + // Returns `true` iff most of HeapProf init process should be skipped due to + // the HeapProf library being loaded via `dlopen()`. Platforms may perform any + // `dlopen()` specific initialization inside this function. + bool HandleDlopenInit(); + +// Add convenient macro for interface functions that may be represented as +// weak hooks. +#define HEAPPROF_MALLOC_HOOK(ptr, size) \ + do { \ + if (&__sanitizer_malloc_hook) \ + __sanitizer_malloc_hook(ptr, size); \ + RunMallocHooks(ptr, size); \ + } while (false) +#define HEAPPROF_FREE_HOOK(ptr) \ + do { \ + if (&__sanitizer_free_hook) \ + __sanitizer_free_hook(ptr); \ + RunFreeHooks(ptr); \ + } while (false) +#define HEAPPROF_ON_ERROR() \ + if (&__heapprof_on_error) \ + __heapprof_on_error() + + extern int heapprof_inited; + extern int heapprof_timestamp_inited; + extern int heapprof_init_done; + // Used to avoid infinite recursion in __heapprof_init(). + extern bool heapprof_init_is_running; + extern void (*death_callback)(void); + extern long heapprof_init_timestamp_s; + +} // namespace __heapprof + +#endif // HEAPPROF_INTERNAL_H diff --git a/compiler-rt/lib/heapprof/heapprof_linux.cpp b/compiler-rt/lib/heapprof/heapprof_linux.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_linux.cpp @@ -0,0 +1,159 @@ +//===-- heapprof_linux.cpp ------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// Linux-specific details. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_platform.h" +#if !SANITIZER_LINUX +#error Unsupported OS +#endif + +#include "heapprof_interceptors.h" +#include "heapprof_internal.h" +#include "heapprof_thread.h" +#include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_freebsd.h" +#include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_procmaps.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +typedef enum { + HEAPPROF_RT_VERSION_UNDEFINED = 0, + HEAPPROF_RT_VERSION_DYNAMIC, + HEAPPROF_RT_VERSION_STATIC, +} heapprof_rt_version_t; + +// FIXME: perhaps also store abi version here? +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE +heapprof_rt_version_t __heapprof_rt_version; +} + +namespace __heapprof { + +void InitializePlatformInterceptors() {} +void InitializePlatformExceptionHandlers() {} +bool IsSystemHeapAddress(uptr addr) { return false; } + +void *HeapprofDoesNotSupportStaticLinkage() { + // This will fail to link with -static. + return &_DYNAMIC; // defined in link.h +} + +uptr FindDynamicShadowStart() { + uptr shadow_size_bytes = MemToShadowSize(kHighMemEnd); + return MapDynamicShadow(shadow_size_bytes, SHADOW_SCALE, + /*min_shadow_base_alignment*/ 0, kHighMemEnd); +} + +static int FindFirstDSOCallback(struct dl_phdr_info *info, size_t size, + void *data) { + VReport(2, "info->dlpi_name = %s\tinfo->dlpi_addr = %p\n", info->dlpi_name, + info->dlpi_addr); + + // Continue until the first dynamic library is found + if (!info->dlpi_name || info->dlpi_name[0] == 0) + return 0; + + // Ignore vDSO + if (internal_strncmp(info->dlpi_name, "linux-", sizeof("linux-") - 1) == 0) + return 0; + + *(const char **)data = info->dlpi_name; + return 1; +} + +static bool IsDynamicRTName(const char *libname) { + return internal_strstr(libname, "libclang_rt.heapprof") || + internal_strstr(libname, "libheapprof.so"); +} + +static void ReportIncompatibleRT() { + Report( + "Your application is linked against incompatible HeapProf runtimes.\n"); + Die(); +} + +void HeapprofCheckDynamicRTPrereqs() { + if (!HEAPPROF_DYNAMIC || !flags()->verify_heapprof_link_order) + return; + + // Ensure that dynamic RT is the first DSO in the list + const char *first_dso_name = nullptr; + dl_iterate_phdr(FindFirstDSOCallback, &first_dso_name); + if (first_dso_name && !IsDynamicRTName(first_dso_name)) { + Report("HeapProf runtime does not come first in initial library list; " + "you should either link runtime to your application or " + "manually preload it with LD_PRELOAD.\n"); + Die(); + } +} + +void HeapprofCheckIncompatibleRT() { + if (HEAPPROF_DYNAMIC) { + if (__heapprof_rt_version == HEAPPROF_RT_VERSION_UNDEFINED) { + __heapprof_rt_version = HEAPPROF_RT_VERSION_DYNAMIC; + } else if (__heapprof_rt_version != HEAPPROF_RT_VERSION_DYNAMIC) { + ReportIncompatibleRT(); + } + } else { + if (__heapprof_rt_version == HEAPPROF_RT_VERSION_UNDEFINED) { + // Ensure that dynamic runtime is not present. We should detect it + // as early as possible, otherwise HeapProf interceptors could bind to + // the functions in dynamic HeapProf runtime instead of the functions in + // system libraries, causing crashes later in HeapProf initialization. + MemoryMappingLayout proc_maps(/*cache_enabled*/ true); + char filename[PATH_MAX]; + MemoryMappedSegment segment(filename, sizeof(filename)); + while (proc_maps.Next(&segment)) { + if (IsDynamicRTName(segment.filename)) { + Report("Your application is linked against " + "incompatible HeapProf runtimes.\n"); + Die(); + } + } + __heapprof_rt_version = HEAPPROF_RT_VERSION_STATIC; + } else if (__heapprof_rt_version != HEAPPROF_RT_VERSION_STATIC) { + ReportIncompatibleRT(); + } + } +} + +void ReadContextStack(void *context, uptr *stack, uptr *ssize) { + ucontext_t *ucp = (ucontext_t *)context; + *stack = (uptr)ucp->uc_stack.ss_sp; + *ssize = ucp->uc_stack.ss_size; +} + +void *HeapprofDlSymNext(const char *sym) { return dlsym(RTLD_NEXT, sym); } + +bool HandleDlopenInit() { + // Not supported on this platform. + static_assert(!SANITIZER_SUPPORTS_INIT_FOR_DLOPEN, + "Expected SANITIZER_SUPPORTS_INIT_FOR_DLOPEN to be false"); + return false; +} + +} // namespace __heapprof diff --git a/compiler-rt/lib/heapprof/heapprof_malloc_linux.cpp b/compiler-rt/lib/heapprof/heapprof_malloc_linux.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_malloc_linux.cpp @@ -0,0 +1,226 @@ +//===-- heapprof_malloc_linux.cpp -----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// Linux-specific malloc interception. +// We simply define functions like malloc, free, realloc, etc. +// They will replace the corresponding libc functions automagically. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_platform.h" +#if !SANITIZER_LINUX +#error Unsupported OS +#endif + +#include "heapprof_allocator.h" +#include "heapprof_interceptors.h" +#include "heapprof_internal.h" +#include "heapprof_stack.h" +#include "sanitizer_common/sanitizer_allocator_checks.h" +#include "sanitizer_common/sanitizer_errno.h" +#include "sanitizer_common/sanitizer_tls_get_addr.h" + +// ---------------------- Replacement functions ---------------- {{{1 +using namespace __heapprof; + +static uptr allocated_for_dlsym; +static uptr last_dlsym_alloc_size_in_words; +static const uptr kDlsymAllocPoolSize = 1024; +static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize]; + +static INLINE bool IsInDlsymAllocPool(const void *ptr) { + uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym; + return off < allocated_for_dlsym * sizeof(alloc_memory_for_dlsym[0]); +} + +static void *AllocateFromLocalPool(uptr size_in_bytes) { + uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize; + void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym]; + last_dlsym_alloc_size_in_words = size_in_words; + allocated_for_dlsym += size_in_words; + CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize); + return mem; +} + +static void DeallocateFromLocalPool(const void *ptr) { + // Hack: since glibc 2.27 dlsym no longer uses stack-allocated memory to store + // error messages and instead uses malloc followed by free. To avoid pool + // exhaustion due to long object filenames, handle that special case here. + uptr prev_offset = allocated_for_dlsym - last_dlsym_alloc_size_in_words; + void *prev_mem = (void *)&alloc_memory_for_dlsym[prev_offset]; + if (prev_mem == ptr) { + REAL(memset)(prev_mem, 0, last_dlsym_alloc_size_in_words * kWordSize); + allocated_for_dlsym = prev_offset; + last_dlsym_alloc_size_in_words = 0; + } +} + +static int PosixMemalignFromLocalPool(void **memptr, uptr alignment, + uptr size_in_bytes) { + if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) + return errno_EINVAL; + + CHECK(alignment >= kWordSize); + + uptr addr = (uptr)&alloc_memory_for_dlsym[allocated_for_dlsym]; + uptr aligned_addr = RoundUpTo(addr, alignment); + uptr aligned_size = RoundUpTo(size_in_bytes, kWordSize); + + uptr *end_mem = (uptr *)(aligned_addr + aligned_size); + uptr allocated = end_mem - alloc_memory_for_dlsym; + if (allocated >= kDlsymAllocPoolSize) + return errno_ENOMEM; + + allocated_for_dlsym = allocated; + *memptr = (void *)aligned_addr; + return 0; +} + +static INLINE bool MaybeInDlsym() { return heapprof_init_is_running; } + +static INLINE bool UseLocalPool() { return MaybeInDlsym(); } + +static void *ReallocFromLocalPool(void *ptr, uptr size) { + const uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym; + const uptr copy_size = Min(size, kDlsymAllocPoolSize - offset); + void *new_ptr; + if (UNLIKELY(UseLocalPool())) { + new_ptr = AllocateFromLocalPool(size); + } else { + ENSURE_HEAPPROF_INITED(); + GET_STACK_TRACE_MALLOC; + new_ptr = heapprof_malloc(size, &stack); + } + internal_memcpy(new_ptr, ptr, copy_size); + return new_ptr; +} + +INTERCEPTOR(void, free, void *ptr) { + GET_STACK_TRACE_FREE; + if (UNLIKELY(IsInDlsymAllocPool(ptr))) { + DeallocateFromLocalPool(ptr); + return; + } + heapprof_free(ptr, &stack, FROM_MALLOC); +} + +#if SANITIZER_INTERCEPT_CFREE +INTERCEPTOR(void, cfree, void *ptr) { + GET_STACK_TRACE_FREE; + if (UNLIKELY(IsInDlsymAllocPool(ptr))) + return; + heapprof_free(ptr, &stack, FROM_MALLOC); +} +#endif // SANITIZER_INTERCEPT_CFREE + +INTERCEPTOR(void *, malloc, uptr size) { + if (UNLIKELY(UseLocalPool())) + // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym. + return AllocateFromLocalPool(size); + ENSURE_HEAPPROF_INITED(); + GET_STACK_TRACE_MALLOC; + return heapprof_malloc(size, &stack); +} + +INTERCEPTOR(void *, calloc, uptr nmemb, uptr size) { + if (UNLIKELY(UseLocalPool())) + // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. + return AllocateFromLocalPool(nmemb * size); + ENSURE_HEAPPROF_INITED(); + GET_STACK_TRACE_MALLOC; + return heapprof_calloc(nmemb, size, &stack); +} + +INTERCEPTOR(void *, realloc, void *ptr, uptr size) { + if (UNLIKELY(IsInDlsymAllocPool(ptr))) + return ReallocFromLocalPool(ptr, size); + if (UNLIKELY(UseLocalPool())) + return AllocateFromLocalPool(size); + ENSURE_HEAPPROF_INITED(); + GET_STACK_TRACE_MALLOC; + return heapprof_realloc(ptr, size, &stack); +} + +#if SANITIZER_INTERCEPT_REALLOCARRAY +INTERCEPTOR(void *, reallocarray, void *ptr, uptr nmemb, uptr size) { + ENSURE_HEAPPROF_INITED(); + GET_STACK_TRACE_MALLOC; + return heapprof_reallocarray(ptr, nmemb, size, &stack); +} +#endif // SANITIZER_INTERCEPT_REALLOCARRAY + +#if SANITIZER_INTERCEPT_MEMALIGN +INTERCEPTOR(void *, memalign, uptr boundary, uptr size) { + GET_STACK_TRACE_MALLOC; + return heapprof_memalign(boundary, size, &stack, FROM_MALLOC); +} + +INTERCEPTOR(void *, __libc_memalign, uptr boundary, uptr size) { + GET_STACK_TRACE_MALLOC; + void *res = heapprof_memalign(boundary, size, &stack, FROM_MALLOC); + DTLS_on_libc_memalign(res, size); + return res; +} +#endif // SANITIZER_INTERCEPT_MEMALIGN + +#if SANITIZER_INTERCEPT_ALIGNED_ALLOC +INTERCEPTOR(void *, aligned_alloc, uptr boundary, uptr size) { + GET_STACK_TRACE_MALLOC; + return heapprof_aligned_alloc(boundary, size, &stack); +} +#endif // SANITIZER_INTERCEPT_ALIGNED_ALLOC + +INTERCEPTOR(uptr, malloc_usable_size, void *ptr) { + GET_CURRENT_PC_BP_SP; + (void)sp; + return heapprof_malloc_usable_size(ptr, pc, bp); +} + +#if SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO +// We avoid including malloc.h for portability reasons. +// man mallinfo says the fields are "long", but the implementation uses int. +// It doesn't matter much -- we just need to make sure that the libc's mallinfo +// is not called. +struct fake_mallinfo { + int x[10]; +}; + +INTERCEPTOR(struct fake_mallinfo, mallinfo, void) { + struct fake_mallinfo res; + REAL(memset)(&res, 0, sizeof(res)); + return res; +} + +INTERCEPTOR(int, mallopt, int cmd, int value) { return 0; } +#endif // SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO + +INTERCEPTOR(int, posix_memalign, void **memptr, uptr alignment, uptr size) { + if (UNLIKELY(UseLocalPool())) + return PosixMemalignFromLocalPool(memptr, alignment, size); + GET_STACK_TRACE_MALLOC; + return heapprof_posix_memalign(memptr, alignment, size, &stack); +} + +INTERCEPTOR(void *, valloc, uptr size) { + GET_STACK_TRACE_MALLOC; + return heapprof_valloc(size, &stack); +} + +#if SANITIZER_INTERCEPT_PVALLOC +INTERCEPTOR(void *, pvalloc, uptr size) { + GET_STACK_TRACE_MALLOC; + return heapprof_pvalloc(size, &stack); +} +#endif // SANITIZER_INTERCEPT_PVALLOC + +INTERCEPTOR(void, malloc_stats, void) { __heapprof_print_accumulated_stats(); } + +namespace __heapprof { +void ReplaceSystemMalloc() {} +} // namespace __heapprof diff --git a/compiler-rt/lib/heapprof/heapprof_mapping.h b/compiler-rt/lib/heapprof/heapprof_mapping.h new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_mapping.h @@ -0,0 +1,143 @@ +//===-- heapprof_mapping.h --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// Defines HeapProf memory mapping. +//===----------------------------------------------------------------------===// +#ifndef HEAPPROF_MAPPING_H +#define HEAPPROF_MAPPING_H + +#include "heapprof_internal.h" + +#if defined(HEAPPROF_SHADOW_SCALE) +static const u64 kDefaultShadowScale = HEAPPROF_SHADOW_SCALE; +#else +static const u64 kDefaultShadowScale = 3; +#endif +#define SHADOW_SCALE kDefaultShadowScale + +#define SHADOW_OFFSET __heapprof_shadow_memory_dynamic_address + +#define SHADOW_GRANULARITY (1ULL << SHADOW_SCALE) +#define HEAPPROF_ALIGNMENT 32 + +#define DO_HEAPPROF_MAPPING_PROFILE \ + 0 // Set to 1 to profile the functions below. + +#if DO_HEAPPROF_MAPPING_PROFILE +#define PROFILE_HEAPPROF_MAPPING() HeapprofMappingProfile[__LINE__]++; +#else +#define PROFILE_HEAPPROF_MAPPING() +#endif + +namespace __heapprof { + +extern uptr HeapprofMappingProfile[]; + +extern uptr kHighMemEnd; // Initialized in __heapprof_init. + +} // namespace __heapprof + +#define SHADOW_ENTRY_SIZE 8 + +// Size of memory block mapped to a single shadow location +#define MEM_GRANULARITY 64ULL + +#define SHADOW_MASK ~(MEM_GRANULARITY - 1) + +#define MEM_TO_SHADOW(mem) \ + (((mem & SHADOW_MASK) >> SHADOW_SCALE) + (SHADOW_OFFSET)) + +#define kLowMemBeg 0 +#define kLowMemEnd (SHADOW_OFFSET ? SHADOW_OFFSET - 1 : 0) + +#define kLowShadowBeg SHADOW_OFFSET +#define kLowShadowEnd (MEM_TO_SHADOW(kLowMemEnd) + SHADOW_ENTRY_SIZE - 1) + +#define kHighMemBeg (MEM_TO_SHADOW(kHighMemEnd) + 1 + SHADOW_ENTRY_SIZE - 1) + +#define kHighShadowBeg MEM_TO_SHADOW(kHighMemBeg) +#define kHighShadowEnd (MEM_TO_SHADOW(kHighMemEnd) + SHADOW_ENTRY_SIZE - 1) + +// With the zero shadow base we can not actually map pages starting from 0. +// This constant is somewhat arbitrary. +#define kZeroBaseShadowStart 0 +#define kZeroBaseMaxShadowStart (1 << 18) + +#define kShadowGapBeg (kLowShadowEnd ? kLowShadowEnd + 1 : kZeroBaseShadowStart) +#define kShadowGapEnd (kHighShadowBeg - 1) + +namespace __heapprof { + +static inline uptr MemToShadowSize(uptr size) { return size >> SHADOW_SCALE; } +static inline bool AddrIsInLowMem(uptr a) { + PROFILE_HEAPPROF_MAPPING(); + return a <= kLowMemEnd; +} + +static inline bool AddrIsInLowShadow(uptr a) { + PROFILE_HEAPPROF_MAPPING(); + return a >= kLowShadowBeg && a <= kLowShadowEnd; +} + +static inline bool AddrIsInHighMem(uptr a) { + PROFILE_HEAPPROF_MAPPING(); + return kHighMemBeg && a >= kHighMemBeg && a <= kHighMemEnd; +} + +static inline bool AddrIsInHighShadow(uptr a) { + PROFILE_HEAPPROF_MAPPING(); + return kHighMemBeg && a >= kHighShadowBeg && a <= kHighShadowEnd; +} + +static inline bool AddrIsInShadowGap(uptr a) { + PROFILE_HEAPPROF_MAPPING(); + // In zero-based shadow mode we treat addresses near zero as addresses + // in shadow gap as well. + if (SHADOW_OFFSET == 0) + return a <= kShadowGapEnd; + return a >= kShadowGapBeg && a <= kShadowGapEnd; +} + +static inline bool AddrIsInMem(uptr a) { + PROFILE_HEAPPROF_MAPPING(); + return AddrIsInLowMem(a) || AddrIsInHighMem(a) || + (flags()->protect_shadow_gap == 0 && AddrIsInShadowGap(a)); +} + +static inline uptr MemToShadow(uptr p) { + PROFILE_HEAPPROF_MAPPING(); + CHECK(AddrIsInMem(p)); + return MEM_TO_SHADOW(p); +} + +static inline bool AddrIsInShadow(uptr a) { + PROFILE_HEAPPROF_MAPPING(); + return AddrIsInLowShadow(a) || AddrIsInHighShadow(a); +} + +static inline bool AddrIsAlignedByGranularity(uptr a) { + PROFILE_HEAPPROF_MAPPING(); + return (a & (SHADOW_GRANULARITY - 1)) == 0; +} + +static inline void RecordAccess(uptr a) { + PROFILE_HEAPPROF_MAPPING(); + // If we use a different shadow size then the type below needs adjustment. + CHECK_EQ(SHADOW_ENTRY_SIZE, 8); + u64 *shadow_address = (u64 *)MEM_TO_SHADOW(a); + (*shadow_address)++; +} + +// Must be after all calls to PROFILE_HEAPPROF_MAPPING(). +static const uptr kHeapprofMappingProfileSize = __LINE__; + +} // namespace __heapprof + +#endif // HEAPPROF_MAPPING_H diff --git a/compiler-rt/lib/heapprof/heapprof_new_delete.cpp b/compiler-rt/lib/heapprof/heapprof_new_delete.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_new_delete.cpp @@ -0,0 +1,145 @@ +//===-- heapprof_interceptors.cpp -----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// Interceptors for operators new and delete. +//===----------------------------------------------------------------------===// + +#include "heapprof_allocator.h" +#include "heapprof_internal.h" +#include "heapprof_report.h" +#include "heapprof_stack.h" + +#include "interception/interception.h" + +#include + +#define CXX_OPERATOR_ATTRIBUTE INTERCEPTOR_ATTRIBUTE + +using namespace __heapprof; + +// Fake std::nothrow_t and std::align_val_t to avoid including . +namespace std { +struct nothrow_t {}; +enum class align_val_t : size_t {}; +} // namespace std + +#define OPERATOR_NEW_BODY(type, nothrow) \ + GET_STACK_TRACE_MALLOC; \ + void *res = heapprof_memalign(0, size, &stack, type); \ + if (!nothrow && UNLIKELY(!res)) \ + ReportOutOfMemory(size, &stack); \ + return res; +#define OPERATOR_NEW_BODY_ALIGN(type, nothrow) \ + GET_STACK_TRACE_MALLOC; \ + void *res = heapprof_memalign((uptr)align, size, &stack, type); \ + if (!nothrow && UNLIKELY(!res)) \ + ReportOutOfMemory(size, &stack); \ + return res; + +CXX_OPERATOR_ATTRIBUTE +void *operator new(size_t size) { + OPERATOR_NEW_BODY(FROM_NEW, false /*nothrow*/); +} +CXX_OPERATOR_ATTRIBUTE +void *operator new[](size_t size) { + OPERATOR_NEW_BODY(FROM_NEW_BR, false /*nothrow*/); +} +CXX_OPERATOR_ATTRIBUTE +void *operator new(size_t size, std::nothrow_t const &) { + OPERATOR_NEW_BODY(FROM_NEW, true /*nothrow*/); +} +CXX_OPERATOR_ATTRIBUTE +void *operator new[](size_t size, std::nothrow_t const &) { + OPERATOR_NEW_BODY(FROM_NEW_BR, true /*nothrow*/); +} +CXX_OPERATOR_ATTRIBUTE +void *operator new(size_t size, std::align_val_t align) { + OPERATOR_NEW_BODY_ALIGN(FROM_NEW, false /*nothrow*/); +} +CXX_OPERATOR_ATTRIBUTE +void *operator new[](size_t size, std::align_val_t align) { + OPERATOR_NEW_BODY_ALIGN(FROM_NEW_BR, false /*nothrow*/); +} +CXX_OPERATOR_ATTRIBUTE +void *operator new(size_t size, std::align_val_t align, + std::nothrow_t const &) { + OPERATOR_NEW_BODY_ALIGN(FROM_NEW, true /*nothrow*/); +} +CXX_OPERATOR_ATTRIBUTE +void *operator new[](size_t size, std::align_val_t align, + std::nothrow_t const &) { + OPERATOR_NEW_BODY_ALIGN(FROM_NEW_BR, true /*nothrow*/); +} + +#define OPERATOR_DELETE_BODY(type) \ + GET_STACK_TRACE_FREE; \ + heapprof_delete(ptr, 0, 0, &stack, type); + +#define OPERATOR_DELETE_BODY_SIZE(type) \ + GET_STACK_TRACE_FREE; \ + heapprof_delete(ptr, size, 0, &stack, type); + +#define OPERATOR_DELETE_BODY_ALIGN(type) \ + GET_STACK_TRACE_FREE; \ + heapprof_delete(ptr, 0, static_cast(align), &stack, type); + +#define OPERATOR_DELETE_BODY_SIZE_ALIGN(type) \ + GET_STACK_TRACE_FREE; \ + heapprof_delete(ptr, size, static_cast(align), &stack, type); + +CXX_OPERATOR_ATTRIBUTE +void operator delete(void *ptr)NOEXCEPT { OPERATOR_DELETE_BODY(FROM_NEW); } +CXX_OPERATOR_ATTRIBUTE +void operator delete[](void *ptr) NOEXCEPT { + OPERATOR_DELETE_BODY(FROM_NEW_BR); +} +CXX_OPERATOR_ATTRIBUTE +void operator delete(void *ptr, std::nothrow_t const &) { + OPERATOR_DELETE_BODY(FROM_NEW); +} +CXX_OPERATOR_ATTRIBUTE +void operator delete[](void *ptr, std::nothrow_t const &) { + OPERATOR_DELETE_BODY(FROM_NEW_BR); +} +CXX_OPERATOR_ATTRIBUTE +void operator delete(void *ptr, size_t size)NOEXCEPT { + OPERATOR_DELETE_BODY_SIZE(FROM_NEW); +} +CXX_OPERATOR_ATTRIBUTE +void operator delete[](void *ptr, size_t size) NOEXCEPT { + OPERATOR_DELETE_BODY_SIZE(FROM_NEW_BR); +} +CXX_OPERATOR_ATTRIBUTE +void operator delete(void *ptr, std::align_val_t align)NOEXCEPT { + OPERATOR_DELETE_BODY_ALIGN(FROM_NEW); +} +CXX_OPERATOR_ATTRIBUTE +void operator delete[](void *ptr, std::align_val_t align) NOEXCEPT { + OPERATOR_DELETE_BODY_ALIGN(FROM_NEW_BR); +} +CXX_OPERATOR_ATTRIBUTE +void operator delete(void *ptr, std::align_val_t align, + std::nothrow_t const &) { + OPERATOR_DELETE_BODY_ALIGN(FROM_NEW); +} +CXX_OPERATOR_ATTRIBUTE +void operator delete[](void *ptr, std::align_val_t align, + std::nothrow_t const &) { + OPERATOR_DELETE_BODY_ALIGN(FROM_NEW_BR); +} +CXX_OPERATOR_ATTRIBUTE +void operator delete(void *ptr, size_t size, std::align_val_t align)NOEXCEPT { + OPERATOR_DELETE_BODY_SIZE_ALIGN(FROM_NEW); +} +CXX_OPERATOR_ATTRIBUTE +void operator delete[](void *ptr, size_t size, + std::align_val_t align) NOEXCEPT { + OPERATOR_DELETE_BODY_SIZE_ALIGN(FROM_NEW_BR); +} diff --git a/compiler-rt/lib/heapprof/heapprof_posix.cpp b/compiler-rt/lib/heapprof/heapprof_posix.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_posix.cpp @@ -0,0 +1,66 @@ +//===-- heapprof_posix.cpp ------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// Posix-specific details. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_platform.h" +#if !SANITIZER_POSIX +#error Only Posix supported +#endif + +#include "heapprof_interceptors.h" +#include "heapprof_internal.h" +#include "heapprof_mapping.h" +#include "heapprof_report.h" +#include "heapprof_stack.h" +#include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_posix.h" +#include "sanitizer_common/sanitizer_procmaps.h" + +#include +#include +#include +#include +#include +#include + +namespace __heapprof { + +// ---------------------- TSD ---------------- {{{1 + +static pthread_key_t tsd_key; +static bool tsd_key_inited = false; +void HeapprofTSDInit(void (*destructor)(void *tsd)) { + CHECK(!tsd_key_inited); + tsd_key_inited = true; + CHECK_EQ(0, pthread_key_create(&tsd_key, destructor)); +} + +void *HeapprofTSDGet() { + CHECK(tsd_key_inited); + return pthread_getspecific(tsd_key); +} + +void HeapprofTSDSet(void *tsd) { + CHECK(tsd_key_inited); + pthread_setspecific(tsd_key, tsd); +} + +void PlatformTSDDtor(void *tsd) { + HeapprofThreadContext *context = (HeapprofThreadContext *)tsd; + if (context->destructor_iterations > 1) { + context->destructor_iterations--; + CHECK_EQ(0, pthread_setspecific(tsd_key, tsd)); + return; + } + HeapprofThread::TSDDtor(tsd); +} +} // namespace __heapprof diff --git a/compiler-rt/lib/heapprof/heapprof_preinit.cpp b/compiler-rt/lib/heapprof/heapprof_preinit.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_preinit.cpp @@ -0,0 +1,23 @@ +//===-- heapprof_preinit.cpp ----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// Call __heapprof_init at the very early stage of process startup. +//===----------------------------------------------------------------------===// +#include "heapprof_internal.h" + +using namespace __heapprof; + +#if SANITIZER_CAN_USE_PREINIT_ARRAY +// The symbol is called __local_heapprof_preinit, because it's not intended to +// be exported. This code linked into the main executable when -fmemory-profile +// is in the link flags. It can only use exported interface functions. +__attribute__((section(".preinit_array"), used)) void ( + *__local_heapprof_preinit)(void) = __heapprof_preinit; +#endif diff --git a/compiler-rt/lib/heapprof/heapprof_report.h b/compiler-rt/lib/heapprof/heapprof_report.h new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_report.h @@ -0,0 +1,40 @@ +//===-- heapprof_report.h ---------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// HeapProf-private header for error reporting functions. +//===----------------------------------------------------------------------===// + +#ifndef HEAPPROF_REPORT_H +#define HEAPPROF_REPORT_H + +#include "heapprof_allocator.h" +#include "heapprof_internal.h" +#include "heapprof_thread.h" + +namespace __heapprof { + +// Different kinds of error reports. +void ReportCallocOverflow(uptr count, uptr size, BufferedStackTrace *stack); +void ReportReallocArrayOverflow(uptr count, uptr size, + BufferedStackTrace *stack); +void ReportPvallocOverflow(uptr size, BufferedStackTrace *stack); +void ReportInvalidAllocationAlignment(uptr alignment, + BufferedStackTrace *stack); +void ReportInvalidAlignedAllocAlignment(uptr size, uptr alignment, + BufferedStackTrace *stack); +void ReportInvalidPosixMemalignAlignment(uptr alignment, + BufferedStackTrace *stack); +void ReportAllocationSizeTooBig(uptr user_size, uptr total_size, uptr max_size, + BufferedStackTrace *stack); +void ReportRssLimitExceeded(BufferedStackTrace *stack); +void ReportOutOfMemory(uptr requested_size, BufferedStackTrace *stack); + +} // namespace __heapprof +#endif // HEAPPROF_REPORT_H diff --git a/compiler-rt/lib/heapprof/heapprof_report.cpp b/compiler-rt/lib/heapprof/heapprof_report.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_report.cpp @@ -0,0 +1,211 @@ +//===-- heapprof_report.cpp -----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// This file contains error reporting code. +//===----------------------------------------------------------------------===// + +#include "heapprof_report.h" +#include "heapprof_descriptions.h" +#include "heapprof_errors.h" +#include "heapprof_flags.h" +#include "heapprof_internal.h" +#include "heapprof_mapping.h" +#include "heapprof_stack.h" +#include "heapprof_thread.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_report_decorator.h" +#include "sanitizer_common/sanitizer_stackdepot.h" +#include "sanitizer_common/sanitizer_symbolizer.h" + +namespace __heapprof { + +// -------------------- User-specified callbacks ----------------- {{{1 +static char *error_message_buffer = nullptr; +static uptr error_message_buffer_pos = 0; +static BlockingMutex error_message_buf_mutex(LINKER_INITIALIZED); + +void AppendToErrorMessageBuffer(const char *buffer) { + BlockingMutexLock l(&error_message_buf_mutex); + if (!error_message_buffer) { + error_message_buffer = + (char *)MmapOrDieQuietly(kErrorMessageBufferSize, __func__); + error_message_buffer_pos = 0; + } + uptr length = internal_strlen(buffer); + RAW_CHECK(kErrorMessageBufferSize >= error_message_buffer_pos); + uptr remaining = kErrorMessageBufferSize - error_message_buffer_pos; + internal_strncpy(error_message_buffer + error_message_buffer_pos, buffer, + remaining); + error_message_buffer[kErrorMessageBufferSize - 1] = '\0'; + // FIXME: reallocate the buffer instead of truncating the message. + error_message_buffer_pos += Min(remaining, length); +} + +// -------------------- Different kinds of reports ----------------- {{{1 + +// Use ScopedInErrorReport to run common actions just before and +// immediately after printing error report. +class ScopedInErrorReport { +public: + explicit ScopedInErrorReport(bool fatal = false) + : halt_on_error_(fatal || flags()->halt_on_error) { + // Make sure the registry and sanitizer report mutexes are locked while + // we're printing an error report. + // We can lock them only here to avoid self-deadlock in case of + // recursive reports. + heapprofThreadRegistry().Lock(); + Printf( + "=================================================================\n"); + } + + ~ScopedInErrorReport() { + if (halt_on_error_ && !__sanitizer_acquire_crash_state()) { + heapprofThreadRegistry().Unlock(); + return; + } + HEAPPROF_ON_ERROR(); + if (current_error_.IsValid()) + current_error_.Print(); + + // Make sure the current thread is announced. + DescribeThread(GetCurrentThread()); + // We may want to grab this lock again when printing stats. + heapprofThreadRegistry().Unlock(); + // Print memory stats. + if (flags()->print_stats) + __heapprof_print_accumulated_stats(); + + if (common_flags()->print_cmdline) + PrintCmdline(); + + if (common_flags()->print_module_map == 2) + PrintModuleMap(); + + // Copy the message buffer so that we could start logging without holding a + // lock that gets aquired during printing. + InternalMmapVector buffer_copy(kErrorMessageBufferSize); + { + BlockingMutexLock l(&error_message_buf_mutex); + internal_memcpy(buffer_copy.data(), error_message_buffer, + kErrorMessageBufferSize); + // Clear error_message_buffer so that if we find other errors + // we don't re-log this error. + error_message_buffer_pos = 0; + } + + LogFullErrorReport(buffer_copy.data()); + + if (halt_on_error_ && common_flags()->abort_on_error) { + // FIXME: implement "compact" error format, possibly without, or with + // highly compressed stack traces? + // FIXME: or just use the summary line as abort message? + SetAbortMessage(buffer_copy.data()); + } + + // In halt_on_error = false mode, reset the current error object (before + // unlocking). + if (!halt_on_error_) + internal_memset(¤t_error_, 0, sizeof(current_error_)); + + if (halt_on_error_) { + Report("ABORTING\n"); + Die(); + } + } + + void ReportError(const ErrorDescription &description) { + // Can only report one error per ScopedInErrorReport. + CHECK_EQ(current_error_.kind, kErrorKindInvalid); + internal_memcpy(¤t_error_, &description, sizeof(current_error_)); + } + + static ErrorDescription &CurrentError() { return current_error_; } + +private: + ScopedErrorReportLock error_report_lock_; + // Error currently being reported. This enables the destructor to interact + // with the debugger and point it to an error description. + static ErrorDescription current_error_; + bool halt_on_error_; +}; + +ErrorDescription ScopedInErrorReport::current_error_(LINKER_INITIALIZED); + +void ReportCallocOverflow(uptr count, uptr size, BufferedStackTrace *stack) { + ScopedInErrorReport in_report(/*fatal*/ true); + ErrorCallocOverflow error(GetCurrentTidOrInvalid(), stack, count, size); + in_report.ReportError(error); +} + +void ReportReallocArrayOverflow(uptr count, uptr size, + BufferedStackTrace *stack) { + ScopedInErrorReport in_report(/*fatal*/ true); + ErrorReallocArrayOverflow error(GetCurrentTidOrInvalid(), stack, count, size); + in_report.ReportError(error); +} + +void ReportPvallocOverflow(uptr size, BufferedStackTrace *stack) { + ScopedInErrorReport in_report(/*fatal*/ true); + ErrorPvallocOverflow error(GetCurrentTidOrInvalid(), stack, size); + in_report.ReportError(error); +} + +void ReportInvalidAllocationAlignment(uptr alignment, + BufferedStackTrace *stack) { + ScopedInErrorReport in_report(/*fatal*/ true); + ErrorInvalidAllocationAlignment error(GetCurrentTidOrInvalid(), stack, + alignment); + in_report.ReportError(error); +} + +void ReportInvalidAlignedAllocAlignment(uptr size, uptr alignment, + BufferedStackTrace *stack) { + ScopedInErrorReport in_report(/*fatal*/ true); + ErrorInvalidAlignedAllocAlignment error(GetCurrentTidOrInvalid(), stack, size, + alignment); + in_report.ReportError(error); +} + +void ReportInvalidPosixMemalignAlignment(uptr alignment, + BufferedStackTrace *stack) { + ScopedInErrorReport in_report(/*fatal*/ true); + ErrorInvalidPosixMemalignAlignment error(GetCurrentTidOrInvalid(), stack, + alignment); + in_report.ReportError(error); +} + +void ReportAllocationSizeTooBig(uptr user_size, uptr total_size, uptr max_size, + BufferedStackTrace *stack) { + ScopedInErrorReport in_report(/*fatal*/ true); + ErrorAllocationSizeTooBig error(GetCurrentTidOrInvalid(), stack, user_size, + total_size, max_size); + in_report.ReportError(error); +} + +void ReportRssLimitExceeded(BufferedStackTrace *stack) { + ScopedInErrorReport in_report(/*fatal*/ true); + ErrorRssLimitExceeded error(GetCurrentTidOrInvalid(), stack); + in_report.ReportError(error); +} + +void ReportOutOfMemory(uptr requested_size, BufferedStackTrace *stack) { + ScopedInErrorReport in_report(/*fatal*/ true); + ErrorOutOfMemory error(GetCurrentTidOrInvalid(), stack, requested_size); + in_report.ReportError(error); +} + +} // namespace __heapprof + +// --------------------------- Interface --------------------- {{{1 + +// Provide default implementation of __heapprof_on_error that does nothing +// and may be overriden by user. +SANITIZER_INTERFACE_WEAK_DEF(void, __heapprof_on_error, void) {} diff --git a/compiler-rt/lib/heapprof/heapprof_rtl.cpp b/compiler-rt/lib/heapprof/heapprof_rtl.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_rtl.cpp @@ -0,0 +1,346 @@ +//===-- heapprof_rtl.cpp --------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// Main file of the HeapProf run-time library. +//===----------------------------------------------------------------------===// + +#include "heapprof_allocator.h" +#include "heapprof_interceptors.h" +#include "heapprof_interface_internal.h" +#include "heapprof_internal.h" +#include "heapprof_mapping.h" +#include "heapprof_report.h" +#include "heapprof_stack.h" +#include "heapprof_stats.h" +#include "heapprof_thread.h" +#include "sanitizer_common/sanitizer_atomic.h" +#include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_symbolizer.h" +#include + +uptr __heapprof_shadow_memory_dynamic_address; // Global interface symbol. + +namespace __heapprof { + +uptr HeapprofMappingProfile[kHeapprofMappingProfileSize]; + +static void HeapprofDie() { + static atomic_uint32_t num_calls; + if (atomic_fetch_add(&num_calls, 1, memory_order_relaxed) != 0) { + // Don't die twice - run a busy loop. + while (1) { + } + } + if (common_flags()->print_module_map >= 1) + PrintModuleMap(); + if (flags()->sleep_before_dying) { + Report("Sleeping for %d second(s)\n", flags()->sleep_before_dying); + SleepForSeconds(flags()->sleep_before_dying); + } + if (flags()->unmap_shadow_on_exit) { + if (kHighShadowEnd) + UnmapOrDie((void *)kLowShadowBeg, kHighShadowEnd - kLowShadowBeg); + } +} + +static void HeapprofCheckFailed(const char *file, int line, const char *cond, + u64 v1, u64 v2) { + Report("HeapProfiler CHECK failed: %s:%d \"%s\" (0x%zx, 0x%zx)\n", file, line, + cond, (uptr)v1, (uptr)v2); + + // Print a stack trace the first time we come here. Otherwise, we probably + // failed a CHECK during symbolization. + static atomic_uint32_t num_calls; + if (atomic_fetch_add(&num_calls, 1, memory_order_relaxed) == 0) { + PRINT_CURRENT_STACK_CHECK(); + } + + Die(); +} + +// -------------------------- Globals --------------------- {{{1 +int heapprof_inited; +int heapprof_init_done; +bool heapprof_init_is_running; +int heapprof_timestamp_inited; +long heapprof_init_timestamp_s; + +uptr kHighMemEnd; + +// -------------------------- Misc ---------------- {{{1 +void ShowStatsAndAbort() { + __heapprof_print_accumulated_stats(); + Die(); +} + +// -------------------------- Run-time entry ------------------- {{{1 +// exported functions + +#define HEAPPROF_MEMORY_ACCESS_CALLBACK_BODY() __heapprof::RecordAccess(addr); + +#define HEAPPROF_MEMORY_ACCESS_CALLBACK(type) \ + extern "C" NOINLINE INTERFACE_ATTRIBUTE void __heapprof_##type(uptr addr) { \ + HEAPPROF_MEMORY_ACCESS_CALLBACK_BODY() \ + } + +HEAPPROF_MEMORY_ACCESS_CALLBACK(load) +HEAPPROF_MEMORY_ACCESS_CALLBACK(store) + +// Force the linker to keep the symbols for various HeapProf interface +// functions. We want to keep those in the executable in order to let the +// instrumented dynamic libraries access the symbol even if it is not used by +// the executable itself. This should help if the build system is removing dead +// code at link time. +static NOINLINE void force_interface_symbols() { + volatile int fake_condition = 0; // prevent dead condition elimination. + // clang-format off + switch (fake_condition) { + case 1: __heapprof_record_access(nullptr); break; + case 2: __heapprof_record_access_range(nullptr,0); break; + } + // clang-format on +} + +static void heapprof_atexit() { + Printf("HeapProfiler exit stats:\n"); + __heapprof_print_accumulated_stats(); + // Print HeapprofMappingProfile. + for (uptr i = 0; i < kHeapprofMappingProfileSize; i++) { + if (HeapprofMappingProfile[i] == 0) + continue; + Printf("heapprof_mapping.h:%zd -- %zd\n", i, HeapprofMappingProfile[i]); + } +} + +static void InitializeHighMemEnd() { + kHighMemEnd = GetMaxUserVirtualAddress(); + // Increase kHighMemEnd to make sure it's properly + // aligned together with kHighMemBeg: + kHighMemEnd |= (GetMmapGranularity() << SHADOW_SCALE) - 1; +} + +void PrintAddressSpaceLayout() { + if (kHighMemBeg) { + Printf("|| `[%p, %p]` || HighMem ||\n", (void *)kHighMemBeg, + (void *)kHighMemEnd); + Printf("|| `[%p, %p]` || HighShadow ||\n", (void *)kHighShadowBeg, + (void *)kHighShadowEnd); + } + Printf("|| `[%p, %p]` || ShadowGap ||\n", (void *)kShadowGapBeg, + (void *)kShadowGapEnd); + if (kLowShadowBeg) { + Printf("|| `[%p, %p]` || LowShadow ||\n", (void *)kLowShadowBeg, + (void *)kLowShadowEnd); + Printf("|| `[%p, %p]` || LowMem ||\n", (void *)kLowMemBeg, + (void *)kLowMemEnd); + } + Printf("MemToShadow(shadow): %p %p", (void *)MEM_TO_SHADOW(kLowShadowBeg), + (void *)MEM_TO_SHADOW(kLowShadowEnd)); + if (kHighMemBeg) { + Printf(" %p %p", (void *)MEM_TO_SHADOW(kHighShadowBeg), + (void *)MEM_TO_SHADOW(kHighShadowEnd)); + } + Printf("\n"); + Printf("malloc_context_size=%zu\n", + (uptr)common_flags()->malloc_context_size); + + Printf("SHADOW_SCALE: %d\n", (int)SHADOW_SCALE); + Printf("SHADOW_GRANULARITY: %d\n", (int)SHADOW_GRANULARITY); + Printf("SHADOW_OFFSET: 0x%zx\n", (uptr)SHADOW_OFFSET); + CHECK(SHADOW_SCALE >= 3 && SHADOW_SCALE <= 7); +} + +static bool UNUSED __local_heapprof_dyninit = [] { + MaybeStartBackgroudThread(); + SetSoftRssLimitExceededCallback(HeapprofSoftRssLimitExceededCallback); + + return false; +}(); + +static void HeapprofInitInternal() { + if (LIKELY(heapprof_inited)) + return; + SanitizerToolName = "HeapProfiler"; + CHECK(!heapprof_init_is_running && "HeapProf init calls itself!"); + heapprof_init_is_running = true; + + CacheBinaryName(); + + // Initialize flags. This must be done early, because most of the + // initialization steps look at flags(). + InitializeFlags(); + + // Stop performing init at this point if we are being loaded via + // dlopen() and the platform supports it. + if (SANITIZER_SUPPORTS_INIT_FOR_DLOPEN && UNLIKELY(HandleDlopenInit())) { + heapprof_init_is_running = false; + VReport(1, "HeapProfiler init is being performed for dlopen().\n"); + return; + } + + HeapprofCheckIncompatibleRT(); + HeapprofCheckDynamicRTPrereqs(); + AvoidCVE_2016_2143(); + + SetMallocContextSize(common_flags()->malloc_context_size); + + InitializeHighMemEnd(); + + // Make sure we are not statically linked. + HeapprofDoesNotSupportStaticLinkage(); + + // Install tool-specific callbacks in sanitizer_common. + AddDieCallback(HeapprofDie); + SetCheckFailedCallback(HeapprofCheckFailed); + SetPrintfAndReportCallback(AppendToErrorMessageBuffer); + + __sanitizer_set_report_path(common_flags()->log_path); + + __sanitizer::InitializePlatformEarly(); + + // Re-exec ourselves if we need to set additional env or command line args. + MaybeReexec(); + + // Setup internal allocator callback. + SetLowLevelAllocateMinAlignment(SHADOW_GRANULARITY); + + InitializeHeapprofInterceptors(); + CheckASLR(); + + ReplaceSystemMalloc(); + + DisableCoreDumperIfNecessary(); + + InitializeShadowMemory(); + + HeapprofTSDInit(PlatformTSDDtor); + + InitializeAllocator(); + + // On Linux HeapprofThread::ThreadStart() calls malloc() that's why + // heapprof_inited should be set to 1 prior to initializing the threads. + heapprof_inited = 1; + heapprof_init_is_running = false; + + if (flags()->atexit) + Atexit(heapprof_atexit); + + InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir); + + // interceptors + InitTlsSize(); + + // Create main thread. + HeapprofThread *main_thread = CreateMainThread(); + CHECK_EQ(0, main_thread->tid()); + force_interface_symbols(); // no-op. + SanitizerInitializeUnwinder(); + + Symbolizer::LateInitialize(); + + VReport(1, "HeapProfiler Init done\n"); + + if (flags()->sleep_after_init) { + Report("Sleeping for %d second(s)\n", flags()->sleep_after_init); + SleepForSeconds(flags()->sleep_after_init); + } + heapprof_init_done = 1; +} + +void HeapprofInitTime() { + if (LIKELY(heapprof_timestamp_inited)) + return; + heapprof_timestamp_inited = 1; + timespec ts; + timespec_get(&ts, TIME_UTC); + heapprof_init_timestamp_s = ts.tv_sec; +} + +// Initialize as requested from some part of HeapProf runtime library +// (interceptors, allocator, etc). +void HeapprofInitFromRtl() { HeapprofInitInternal(); } + +#if HEAPPROF_DYNAMIC +// Initialize runtime in case it's LD_PRELOAD-ed into unsanitized executable +// (and thus normal initializers from .preinit_array or modules haven't run). + +class HeapprofInitializer { +public: + HeapprofInitializer() { HeapprofInitFromRtl(); } +}; + +static HeapprofInitializer heapprof_initializer; +#endif // HEAPPROF_DYNAMIC + +} // namespace __heapprof + +// ---------------------- Interface ---------------- {{{1 +using namespace __heapprof; + +// Initialize as requested from instrumented application code. +void __heapprof_init() { + HeapprofInitTime(); + HeapprofInitInternal(); +} + +void __heapprof_preinit() { HeapprofInitInternal(); } + +void __heapprof_version_mismatch_check_v1() {} + +void __heapprof_record_access(void const volatile *addr) { + __heapprof::RecordAccess((uptr)addr); +} + +// We only record the access on the first location in the range, +// since we will later accumulate the access counts across the +// full allocation, and we don't want to inflate the hotness from +// a memory intrinsic on a large range of memory. +// TODO: Should we do something else so we can better track utilization? +void __heapprof_record_access_range(void const volatile *addr, + UNUSED uptr size) { + __heapprof::RecordAccess((uptr)addr); +} + +extern "C" SANITIZER_INTERFACE_ATTRIBUTE u16 +__sanitizer_unaligned_load16(const uu16 *p) { + __heapprof_record_access(p); + return *p; +} + +extern "C" SANITIZER_INTERFACE_ATTRIBUTE u32 +__sanitizer_unaligned_load32(const uu32 *p) { + __heapprof_record_access(p); + return *p; +} + +extern "C" SANITIZER_INTERFACE_ATTRIBUTE u64 +__sanitizer_unaligned_load64(const uu64 *p) { + __heapprof_record_access(p); + return *p; +} + +extern "C" SANITIZER_INTERFACE_ATTRIBUTE void +__sanitizer_unaligned_store16(uu16 *p, u16 x) { + __heapprof_record_access(p); + *p = x; +} + +extern "C" SANITIZER_INTERFACE_ATTRIBUTE void +__sanitizer_unaligned_store32(uu32 *p, u32 x) { + __heapprof_record_access(p); + *p = x; +} + +extern "C" SANITIZER_INTERFACE_ATTRIBUTE void +__sanitizer_unaligned_store64(uu64 *p, u64 x) { + __heapprof_record_access(p); + *p = x; +} diff --git a/compiler-rt/lib/heapprof/heapprof_shadow_setup.cpp b/compiler-rt/lib/heapprof/heapprof_shadow_setup.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_shadow_setup.cpp @@ -0,0 +1,62 @@ +//===-- heapprof_shadow_setup.cpp -----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// Set up the shadow memory. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_platform.h" + +#include "heapprof_internal.h" +#include "heapprof_mapping.h" + +namespace __heapprof { + +static void ProtectGap(uptr addr, uptr size) { + if (!flags()->protect_shadow_gap) { + // The shadow gap is unprotected, so there is a chance that someone + // is actually using this memory. Which means it needs a shadow... + uptr GapShadowBeg = RoundDownTo(MEM_TO_SHADOW(addr), GetPageSizeCached()); + uptr GapShadowEnd = + RoundUpTo(MEM_TO_SHADOW(addr + size), GetPageSizeCached()) - 1; + if (Verbosity()) + Printf("protect_shadow_gap=0:" + " not protecting shadow gap, allocating gap's shadow\n" + "|| `[%p, %p]` || ShadowGap's shadow ||\n", + GapShadowBeg, GapShadowEnd); + ReserveShadowMemoryRange(GapShadowBeg, GapShadowEnd, + "unprotected gap shadow"); + return; + } + __sanitizer::ProtectGap(addr, size, kZeroBaseShadowStart, + kZeroBaseMaxShadowStart); +} + +void InitializeShadowMemory() { + uptr shadow_start = FindDynamicShadowStart(); + // Update the shadow memory address (potentially) used by instrumentation. + __heapprof_shadow_memory_dynamic_address = shadow_start; + + if (kLowShadowBeg) + shadow_start -= GetMmapGranularity(); + + if (Verbosity()) + PrintAddressSpaceLayout(); + + // mmap the low shadow plus at least one page at the left. + if (kLowShadowBeg) + ReserveShadowMemoryRange(shadow_start, kLowShadowEnd, "low shadow"); + // mmap the high shadow. + ReserveShadowMemoryRange(kHighShadowBeg, kHighShadowEnd, "high shadow"); + // protect the gap. + ProtectGap(kShadowGapBeg, kShadowGapEnd - kShadowGapBeg + 1); + CHECK_EQ(kShadowGapEnd, kHighShadowBeg - 1); +} + +} // namespace __heapprof diff --git a/compiler-rt/lib/heapprof/heapprof_stack.h b/compiler-rt/lib/heapprof/heapprof_stack.h new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_stack.h @@ -0,0 +1,84 @@ +//===-- heapprof_stack.h ----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// HeapProf-private header for heapprof_stack.cpp. +//===----------------------------------------------------------------------===// + +#ifndef HEAPPROF_STACK_H +#define HEAPPROF_STACK_H + +#include "heapprof_flags.h" +#include "heapprof_thread.h" +#include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_stacktrace.h" + +namespace __heapprof { + +static const u32 kDefaultMallocContextSize = 30; + +void SetMallocContextSize(u32 size); +u32 GetMallocContextSize(); + +} // namespace __heapprof + +// NOTE: A Rule of thumb is to retrieve stack trace in the interceptors +// as early as possible (in functions exposed to the user), as we generally +// don't want stack trace to contain functions from HeapProf internals. + +#define GET_STACK_TRACE(max_size, fast) \ + BufferedStackTrace stack; \ + if (max_size <= 2) { \ + stack.size = max_size; \ + if (max_size > 0) { \ + stack.top_frame_bp = GET_CURRENT_FRAME(); \ + stack.trace_buffer[0] = StackTrace::GetCurrentPc(); \ + if (max_size > 1) \ + stack.trace_buffer[1] = GET_CALLER_PC(); \ + } \ + } else { \ + stack.Unwind(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), nullptr, \ + fast, max_size); \ + } + +#define GET_STACK_TRACE_FATAL(pc, bp) \ + BufferedStackTrace stack; \ + stack.Unwind(pc, bp, nullptr, common_flags()->fast_unwind_on_fatal) + +#define GET_STACK_TRACE_SIGNAL(sig) \ + BufferedStackTrace stack; \ + stack.Unwind((sig).pc, (sig).bp, (sig).context, \ + common_flags()->fast_unwind_on_fatal) + +#define GET_STACK_TRACE_FATAL_HERE \ + GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_fatal) + +#define GET_STACK_TRACE_CHECK_HERE \ + GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_check) + +#define GET_STACK_TRACE_THREAD GET_STACK_TRACE(kStackTraceMax, true) + +#define GET_STACK_TRACE_MALLOC \ + GET_STACK_TRACE(GetMallocContextSize(), common_flags()->fast_unwind_on_malloc) + +#define GET_STACK_TRACE_FREE GET_STACK_TRACE_MALLOC + +#define PRINT_CURRENT_STACK() \ + { \ + GET_STACK_TRACE_FATAL_HERE; \ + stack.Print(); \ + } + +#define PRINT_CURRENT_STACK_CHECK() \ + { \ + GET_STACK_TRACE_CHECK_HERE; \ + stack.Print(); \ + } + +#endif // HEAPPROF_STACK_H diff --git a/compiler-rt/lib/heapprof/heapprof_stack.cpp b/compiler-rt/lib/heapprof/heapprof_stack.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_stack.cpp @@ -0,0 +1,87 @@ +//===-- heapprof_stack.cpp ------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// Code for HeapProf stack trace. +//===----------------------------------------------------------------------===// +#include "heapprof_stack.h" +#include "heapprof_internal.h" +#include "sanitizer_common/sanitizer_atomic.h" + +namespace __heapprof { + +static atomic_uint32_t malloc_context_size; + +void SetMallocContextSize(u32 size) { + atomic_store(&malloc_context_size, size, memory_order_release); +} + +u32 GetMallocContextSize() { + return atomic_load(&malloc_context_size, memory_order_acquire); +} + +namespace { + +// ScopedUnwinding is a scope for stacktracing member of a context +class ScopedUnwinding { +public: + explicit ScopedUnwinding(HeapprofThread *t) : thread(t) { + if (thread) { + can_unwind = !thread->isUnwinding(); + thread->setUnwinding(true); + } + } + ~ScopedUnwinding() { + if (thread) + thread->setUnwinding(false); + } + + bool CanUnwind() const { return can_unwind; } + +private: + HeapprofThread *thread = nullptr; + bool can_unwind = true; +}; + +} // namespace + +} // namespace __heapprof + +void __sanitizer::BufferedStackTrace::UnwindImpl(uptr pc, uptr bp, + void *context, + bool request_fast, + u32 max_depth) { + using namespace __heapprof; + size = 0; + if (UNLIKELY(!heapprof_inited)) + return; + request_fast = StackTrace::WillUseFastUnwind(request_fast); + HeapprofThread *t = GetCurrentThread(); + ScopedUnwinding unwind_scope(t); + if (!unwind_scope.CanUnwind()) + return; + if (request_fast) { + if (t) { + Unwind(max_depth, pc, bp, nullptr, t->stack_top(), t->stack_bottom(), + true); + } + return; + } + Unwind(max_depth, pc, bp, context, 0, 0, false); +} + +// ------------------ Interface -------------- {{{1 + +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_print_stack_trace() { + using namespace __heapprof; + PRINT_CURRENT_STACK(); +} +} // extern "C" diff --git a/compiler-rt/lib/heapprof/heapprof_stats.h b/compiler-rt/lib/heapprof/heapprof_stats.h new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_stats.h @@ -0,0 +1,61 @@ +//===-- heapprof_stats.h ----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// HeapProf-private header for statistics. +//===----------------------------------------------------------------------===// +#ifndef HEAPPROF_STATS_H +#define HEAPPROF_STATS_H + +#include "heapprof_allocator.h" +#include "heapprof_internal.h" + +namespace __heapprof { + +// HeapprofStats struct is NOT thread-safe. +// Each HeapprofThread has its own HeapprofStats, which are sometimes flushed +// to the accumulated HeapprofStats. +struct HeapprofStats { + // HeapprofStats must be a struct consisting of uptr fields only. + // When merging two HeapprofStats structs, we treat them as arrays of uptr. + uptr mallocs; + uptr malloced; + uptr malloced_overhead; + uptr frees; + uptr freed; + uptr real_frees; + uptr really_freed; + uptr reallocs; + uptr realloced; + uptr mmaps; + uptr mmaped; + uptr munmaps; + uptr munmaped; + uptr malloc_large; + uptr malloced_by_size[kNumberOfSizeClasses]; + + // Ctor for global HeapprofStats (accumulated stats for dead threads). + explicit HeapprofStats(LinkerInitialized) {} + // Creates empty stats. + HeapprofStats(); + + void Print(); // Prints formatted stats to stderr. + void Clear(); + void MergeFrom(const HeapprofStats *stats); +}; + +// Returns stats for GetCurrentThread(), or stats for fake "unknown thread" +// if GetCurrentThread() returns 0. +HeapprofStats &GetCurrentThreadStats(); +// Flushes a given stats into accumulated stats of dead threads. +void FlushToDeadThreadStats(HeapprofStats *stats); + +} // namespace __heapprof + +#endif // HEAPPROF_STATS_H diff --git a/compiler-rt/lib/heapprof/heapprof_stats.cpp b/compiler-rt/lib/heapprof/heapprof_stats.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_stats.cpp @@ -0,0 +1,127 @@ +//===-- heapprof_stats.cpp ------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// Code related to statistics collected by HeapProfiler. +//===----------------------------------------------------------------------===// +#include "heapprof_stats.h" +#include "heapprof_interceptors.h" +#include "heapprof_internal.h" +#include "heapprof_thread.h" +#include "sanitizer_common/sanitizer_mutex.h" +#include "sanitizer_common/sanitizer_stackdepot.h" + +namespace __heapprof { + +HeapprofStats::HeapprofStats() { Clear(); } + +void HeapprofStats::Clear() { + CHECK(REAL(memset)); + REAL(memset)(this, 0, sizeof(HeapprofStats)); +} + +static void PrintMallocStatsArray(const char *prefix, + uptr (&array)[kNumberOfSizeClasses]) { + Printf("%s", prefix); + for (uptr i = 0; i < kNumberOfSizeClasses; i++) { + if (!array[i]) + continue; + Printf("%zu:%zu; ", i, array[i]); + } + Printf("\n"); +} + +void HeapprofStats::Print() { + Printf("Stats: %zuM malloced (%zuM for overhead) by %zu calls\n", + malloced >> 20, malloced_overhead >> 20, mallocs); + Printf("Stats: %zuM realloced by %zu calls\n", realloced >> 20, reallocs); + Printf("Stats: %zuM freed by %zu calls\n", freed >> 20, frees); + Printf("Stats: %zuM really freed by %zu calls\n", really_freed >> 20, + real_frees); + Printf("Stats: %zuM (%zuM-%zuM) mmaped; %zu maps, %zu unmaps\n", + (mmaped - munmaped) >> 20, mmaped >> 20, munmaped >> 20, mmaps, + munmaps); + + PrintMallocStatsArray(" mallocs by size class: ", malloced_by_size); + Printf("Stats: malloc large: %zu\n", malloc_large); +} + +void HeapprofStats::MergeFrom(const HeapprofStats *stats) { + uptr *dst_ptr = reinterpret_cast(this); + const uptr *src_ptr = reinterpret_cast(stats); + uptr num_fields = sizeof(*this) / sizeof(uptr); + for (uptr i = 0; i < num_fields; i++) + dst_ptr[i] += src_ptr[i]; +} + +static BlockingMutex print_lock(LINKER_INITIALIZED); + +static HeapprofStats unknown_thread_stats(LINKER_INITIALIZED); +static HeapprofStats dead_threads_stats(LINKER_INITIALIZED); +static BlockingMutex dead_threads_stats_lock(LINKER_INITIALIZED); +// Required for malloc_zone_statistics() on OS X. This can't be stored in +// per-thread HeapprofStats. +static uptr max_malloced_memory; + +static void MergeThreadStats(ThreadContextBase *tctx_base, void *arg) { + HeapprofStats *accumulated_stats = reinterpret_cast(arg); + HeapprofThreadContext *tctx = static_cast(tctx_base); + if (HeapprofThread *t = tctx->thread) + accumulated_stats->MergeFrom(&t->stats()); +} + +static void GetAccumulatedStats(HeapprofStats *stats) { + stats->Clear(); + { + ThreadRegistryLock l(&heapprofThreadRegistry()); + heapprofThreadRegistry().RunCallbackForEachThreadLocked(MergeThreadStats, + stats); + } + stats->MergeFrom(&unknown_thread_stats); + { + BlockingMutexLock lock(&dead_threads_stats_lock); + stats->MergeFrom(&dead_threads_stats); + } + // This is not very accurate: we may miss allocation peaks that happen + // between two updates of accumulated_stats_. For more accurate bookkeeping + // the maximum should be updated on every malloc(), which is unacceptable. + if (max_malloced_memory < stats->malloced) { + max_malloced_memory = stats->malloced; + } +} + +void FlushToDeadThreadStats(HeapprofStats *stats) { + BlockingMutexLock lock(&dead_threads_stats_lock); + dead_threads_stats.MergeFrom(stats); + stats->Clear(); +} + +HeapprofStats &GetCurrentThreadStats() { + HeapprofThread *t = GetCurrentThread(); + return (t) ? t->stats() : unknown_thread_stats; +} + +static void PrintAccumulatedStats() { + HeapprofStats stats; + GetAccumulatedStats(&stats); + // Use lock to keep reports from mixing up. + BlockingMutexLock lock(&print_lock); + stats.Print(); + StackDepotStats *stack_depot_stats = StackDepotGetStats(); + Printf("Stats: StackDepot: %zd ids; %zdM allocated\n", + stack_depot_stats->n_uniq_ids, stack_depot_stats->allocated >> 20); + PrintInternalAllocatorStats(); +} + +} // namespace __heapprof + +// ---------------------- Interface ---------------- {{{1 +using namespace __heapprof; + +void __heapprof_print_accumulated_stats() { PrintAccumulatedStats(); } diff --git a/compiler-rt/lib/heapprof/heapprof_thread.h b/compiler-rt/lib/heapprof/heapprof_thread.h new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_thread.h @@ -0,0 +1,147 @@ +//===-- heapprof_thread.h ---------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// HeapProf-private header for heapprof_thread.cpp. +//===----------------------------------------------------------------------===// + +#ifndef HEAPPROF_THREAD_H +#define HEAPPROF_THREAD_H + +#include "heapprof_allocator.h" +#include "heapprof_internal.h" +#include "heapprof_stats.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_thread_registry.h" + +namespace __sanitizer { +struct DTLS; +} // namespace __sanitizer + +namespace __heapprof { + +const u32 kInvalidTid = 0xffffff; // Must fit into 24 bits. +const u32 kMaxNumberOfThreads = (1 << 22); // 4M + +class HeapprofThread; + +// These objects are created for every thread and are never deleted, +// so we can find them by tid even if the thread is long dead. +class HeapprofThreadContext : public ThreadContextBase { +public: + explicit HeapprofThreadContext(int tid) + : ThreadContextBase(tid), announced(false), + destructor_iterations(GetPthreadDestructorIterations()), stack_id(0), + thread(nullptr) {} + bool announced; + u8 destructor_iterations; + u32 stack_id; + HeapprofThread *thread; + + void OnCreated(void *arg) override; + void OnFinished() override; + + struct CreateThreadContextArgs { + HeapprofThread *thread; + StackTrace *stack; + }; +}; + +// HeapprofThreadContext objects are never freed, so we need many of them. +COMPILER_CHECK(sizeof(HeapprofThreadContext) <= 256); + +// HeapprofThread are stored in TSD and destroyed when the thread dies. +class HeapprofThread { +public: + static HeapprofThread *Create(thread_callback_t start_routine, void *arg, + u32 parent_tid, StackTrace *stack, + bool detached); + static void TSDDtor(void *tsd); + void Destroy(); + + struct InitOptions; + void Init(const InitOptions *options = nullptr); + + thread_return_t ThreadStart(tid_t os_id, + atomic_uintptr_t *signal_thread_is_registered); + + uptr stack_top(); + uptr stack_bottom(); + uptr stack_size(); + uptr tls_begin() { return tls_begin_; } + uptr tls_end() { return tls_end_; } + DTLS *dtls() { return dtls_; } + u32 tid() { return context_->tid; } + HeapprofThreadContext *context() { return context_; } + void set_context(HeapprofThreadContext *context) { context_ = context; } + + bool AddrIsInStack(uptr addr); + + void StartSwitchFiber(uptr bottom, uptr size); + void FinishSwitchFiber(uptr *bottom_old, uptr *size_old); + + // True is this thread is currently unwinding stack (i.e. collecting a stack + // trace). Used to prevent deadlocks on platforms where libc unwinder calls + // malloc internally. See PR17116 for more details. + bool isUnwinding() const { return unwinding_; } + void setUnwinding(bool b) { unwinding_ = b; } + + HeapprofThreadLocalMallocStorage &malloc_storage() { return malloc_storage_; } + HeapprofStats &stats() { return stats_; } + +private: + // NOTE: There is no HeapprofThread constructor. It is allocated + // via mmap() and *must* be valid in zero-initialized state. + + void SetThreadStackAndTls(const InitOptions *options); + + struct StackBounds { + uptr bottom; + uptr top; + }; + StackBounds GetStackBounds() const; + + HeapprofThreadContext *context_; + thread_callback_t start_routine_; + void *arg_; + + uptr stack_top_; + uptr stack_bottom_; + // these variables are used when the thread is about to switch stack + uptr next_stack_top_; + uptr next_stack_bottom_; + // true if switching is in progress + atomic_uint8_t stack_switching_; + + uptr tls_begin_; + uptr tls_end_; + DTLS *dtls_; + + HeapprofThreadLocalMallocStorage malloc_storage_; + HeapprofStats stats_; + bool unwinding_; +}; + +// Returns a single instance of registry. +ThreadRegistry &heapprofThreadRegistry(); + +// Must be called under ThreadRegistryLock. +HeapprofThreadContext *GetThreadContextByTidLocked(u32 tid); + +// Get the current thread. May return 0. +HeapprofThread *GetCurrentThread(); +void SetCurrentThread(HeapprofThread *t); +u32 GetCurrentTidOrInvalid(); + +// Used to handle fork(). +void EnsureMainThreadIDIsCorrect(); +} // namespace __heapprof + +#endif // HEAPPROF_THREAD_H diff --git a/compiler-rt/lib/heapprof/heapprof_thread.cpp b/compiler-rt/lib/heapprof/heapprof_thread.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/heapprof_thread.cpp @@ -0,0 +1,293 @@ +//===-- heapprof_thread.cpp -----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HeapProfiler, a memory profiler. +// +// Thread-related code. +//===----------------------------------------------------------------------===// +#include "heapprof_thread.h" +#include "heapprof_allocator.h" +#include "heapprof_interceptors.h" +#include "heapprof_mapping.h" +#include "heapprof_stack.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_placement_new.h" +#include "sanitizer_common/sanitizer_stackdepot.h" +#include "sanitizer_common/sanitizer_tls_get_addr.h" + +namespace __heapprof { + +// HeapprofThreadContext implementation. + +void HeapprofThreadContext::OnCreated(void *arg) { + CreateThreadContextArgs *args = static_cast(arg); + if (args->stack) + stack_id = StackDepotPut(*args->stack); + thread = args->thread; + thread->set_context(this); +} + +void HeapprofThreadContext::OnFinished() { + // Drop the link to the HeapprofThread object. + thread = nullptr; +} + +static ALIGNED(16) char thread_registry_placeholder[sizeof(ThreadRegistry)]; +static ThreadRegistry *heapprof_thread_registry; + +static BlockingMutex mu_for_thread_context(LINKER_INITIALIZED); +static LowLevelAllocator allocator_for_thread_context; + +static ThreadContextBase *GetHeapprofThreadContext(u32 tid) { + BlockingMutexLock lock(&mu_for_thread_context); + return new (allocator_for_thread_context) HeapprofThreadContext(tid); +} + +ThreadRegistry &heapprofThreadRegistry() { + static bool initialized; + // Don't worry about thread_safety - this should be called when there is + // a single thread. + if (!initialized) { + // Never reuse HeapProf threads: we store pointer to HeapprofThreadContext + // in TSD and can't reliably tell when no more TSD destructors will + // be called. It would be wrong to reuse HeapprofThreadContext for another + // thread before all TSD destructors will be called for it. + heapprof_thread_registry = new (thread_registry_placeholder) ThreadRegistry( + GetHeapprofThreadContext, kMaxNumberOfThreads, kMaxNumberOfThreads); + initialized = true; + } + return *heapprof_thread_registry; +} + +HeapprofThreadContext *GetThreadContextByTidLocked(u32 tid) { + return static_cast( + heapprofThreadRegistry().GetThreadLocked(tid)); +} + +// HeapprofThread implementation. + +HeapprofThread *HeapprofThread::Create(thread_callback_t start_routine, + void *arg, u32 parent_tid, + StackTrace *stack, bool detached) { + uptr PageSize = GetPageSizeCached(); + uptr size = RoundUpTo(sizeof(HeapprofThread), PageSize); + HeapprofThread *thread = (HeapprofThread *)MmapOrDie(size, __func__); + thread->start_routine_ = start_routine; + thread->arg_ = arg; + HeapprofThreadContext::CreateThreadContextArgs args = {thread, stack}; + heapprofThreadRegistry().CreateThread(*reinterpret_cast(thread), + detached, parent_tid, &args); + + return thread; +} + +void HeapprofThread::TSDDtor(void *tsd) { + HeapprofThreadContext *context = (HeapprofThreadContext *)tsd; + VReport(1, "T%d TSDDtor\n", context->tid); + if (context->thread) + context->thread->Destroy(); +} + +void HeapprofThread::Destroy() { + int tid = this->tid(); + VReport(1, "T%d exited\n", tid); + + malloc_storage().CommitBack(); + if (common_flags()->use_sigaltstack) + UnsetAlternateSignalStack(); + heapprofThreadRegistry().FinishThread(tid); + FlushToDeadThreadStats(&stats_); + uptr size = RoundUpTo(sizeof(HeapprofThread), GetPageSizeCached()); + UnmapOrDie(this, size); + DTLS_Destroy(); +} + +void HeapprofThread::StartSwitchFiber(uptr bottom, uptr size) { + if (atomic_load(&stack_switching_, memory_order_relaxed)) { + Report("ERROR: starting fiber switch while in fiber switch\n"); + Die(); + } + + next_stack_bottom_ = bottom; + next_stack_top_ = bottom + size; + atomic_store(&stack_switching_, 1, memory_order_release); +} + +void HeapprofThread::FinishSwitchFiber(uptr *bottom_old, uptr *size_old) { + if (!atomic_load(&stack_switching_, memory_order_relaxed)) { + Report("ERROR: finishing a fiber switch that has not started\n"); + Die(); + } + + if (bottom_old) + *bottom_old = stack_bottom_; + if (size_old) + *size_old = stack_top_ - stack_bottom_; + stack_bottom_ = next_stack_bottom_; + stack_top_ = next_stack_top_; + atomic_store(&stack_switching_, 0, memory_order_release); + next_stack_top_ = 0; + next_stack_bottom_ = 0; +} + +inline HeapprofThread::StackBounds HeapprofThread::GetStackBounds() const { + if (!atomic_load(&stack_switching_, memory_order_acquire)) { + // Make sure the stack bounds are fully initialized. + if (stack_bottom_ >= stack_top_) + return {0, 0}; + return {stack_bottom_, stack_top_}; + } + char local; + const uptr cur_stack = (uptr)&local; + // Note: need to check next stack first, because FinishSwitchFiber + // may be in process of overwriting stack_top_/bottom_. But in such case + // we are already on the next stack. + if (cur_stack >= next_stack_bottom_ && cur_stack < next_stack_top_) + return {next_stack_bottom_, next_stack_top_}; + return {stack_bottom_, stack_top_}; +} + +uptr HeapprofThread::stack_top() { return GetStackBounds().top; } + +uptr HeapprofThread::stack_bottom() { return GetStackBounds().bottom; } + +uptr HeapprofThread::stack_size() { + const auto bounds = GetStackBounds(); + return bounds.top - bounds.bottom; +} + +void HeapprofThread::Init(const InitOptions *options) { + next_stack_top_ = next_stack_bottom_ = 0; + atomic_store(&stack_switching_, false, memory_order_release); + CHECK_EQ(this->stack_size(), 0U); + SetThreadStackAndTls(options); + if (stack_top_ != stack_bottom_) { + CHECK_GT(this->stack_size(), 0U); + CHECK(AddrIsInMem(stack_bottom_)); + CHECK(AddrIsInMem(stack_top_ - 1)); + } + int local = 0; + VReport(1, "T%d: stack [%p,%p) size 0x%zx; local=%p\n", tid(), + (void *)stack_bottom_, (void *)stack_top_, stack_top_ - stack_bottom_, + &local); +} + +thread_return_t +HeapprofThread::ThreadStart(tid_t os_id, + atomic_uintptr_t *signal_thread_is_registered) { + Init(); + heapprofThreadRegistry().StartThread(tid(), os_id, ThreadType::Regular, + nullptr); + if (signal_thread_is_registered) + atomic_store(signal_thread_is_registered, 1, memory_order_release); + + if (common_flags()->use_sigaltstack) + SetAlternateSignalStack(); + + if (!start_routine_) { + // start_routine_ == 0 if we're on the main thread or on one of the + // OS X libdispatch worker threads. But nobody is supposed to call + // ThreadStart() for the worker threads. + CHECK_EQ(tid(), 0); + return 0; + } + + return start_routine_(arg_); +} + +HeapprofThread *CreateMainThread() { + HeapprofThread *main_thread = HeapprofThread::Create( + /* start_routine */ nullptr, /* arg */ nullptr, /* parent_tid */ 0, + /* stack */ nullptr, /* detached */ true); + SetCurrentThread(main_thread); + main_thread->ThreadStart(internal_getpid(), + /* signal_thread_is_registered */ nullptr); + return main_thread; +} + +// This implementation doesn't use the argument, which is just passed down +// from the caller of Init (which see, above). It's only there to support +// OS-specific implementations that need more information passed through. +void HeapprofThread::SetThreadStackAndTls(const InitOptions *options) { + DCHECK_EQ(options, nullptr); + uptr tls_size = 0; + uptr stack_size = 0; + GetThreadStackAndTls(tid() == 0, &stack_bottom_, &stack_size, &tls_begin_, + &tls_size); + stack_top_ = stack_bottom_ + stack_size; + tls_end_ = tls_begin_ + tls_size; + dtls_ = DTLS_Get(); + + if (stack_top_ != stack_bottom_) { + int local; + CHECK(AddrIsInStack((uptr)&local)); + } +} + +bool HeapprofThread::AddrIsInStack(uptr addr) { + const auto bounds = GetStackBounds(); + return addr >= bounds.bottom && addr < bounds.top; +} + +HeapprofThread *GetCurrentThread() { + HeapprofThreadContext *context = + reinterpret_cast(HeapprofTSDGet()); + if (!context) + return nullptr; + return context->thread; +} + +void SetCurrentThread(HeapprofThread *t) { + CHECK(t->context()); + VReport(2, "SetCurrentThread: %p for thread %p\n", t->context(), + (void *)GetThreadSelf()); + // Make sure we do not reset the current HeapprofThread. + CHECK_EQ(0, HeapprofTSDGet()); + HeapprofTSDSet(t->context()); + CHECK_EQ(t->context(), HeapprofTSDGet()); +} + +u32 GetCurrentTidOrInvalid() { + HeapprofThread *t = GetCurrentThread(); + return t ? t->tid() : kInvalidTid; +} + +void EnsureMainThreadIDIsCorrect() { + HeapprofThreadContext *context = + reinterpret_cast(HeapprofTSDGet()); + if (context && (context->tid == 0)) + context->os_id = GetTid(); +} +} // namespace __heapprof + +// ---------------------- Interface ---------------- {{{1 +using namespace __heapprof; + +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_start_switch_fiber(UNUSED void **fakestacksave, + const void *bottom, uptr size) { + HeapprofThread *t = GetCurrentThread(); + if (!t) { + VReport(1, "__heapprof_start_switch_fiber called from unknown thread\n"); + return; + } + t->StartSwitchFiber((uptr)bottom, size); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_finish_switch_fiber(UNUSED void *fakestack, + const void **bottom_old, uptr *size_old) { + HeapprofThread *t = GetCurrentThread(); + if (!t) { + VReport(1, "__heapprof_finish_switch_fiber called from unknown thread\n"); + return; + } + t->FinishSwitchFiber((uptr *)bottom_old, (uptr *)size_old); +} +} diff --git a/compiler-rt/lib/heapprof/weak_symbols.txt b/compiler-rt/lib/heapprof/weak_symbols.txt new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/heapprof/weak_symbols.txt @@ -0,0 +1 @@ +___heapprof_default_options ___heapprof_on_error diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h b/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h --- a/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h @@ -455,5 +455,8 @@ namespace __hwasan { using namespace __sanitizer; } +namespace __heapprof { +using namespace __sanitizer; +} #endif // SANITIZER_DEFS_H diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_libc.h b/compiler-rt/lib/sanitizer_common/sanitizer_libc.h --- a/compiler-rt/lib/sanitizer_common/sanitizer_libc.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_libc.h @@ -71,6 +71,7 @@ uptr internal_getpid(); uptr internal_getppid(); +uptr internal_getcpu(); int internal_dlinfo(void *handle, int request, void *p); diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp --- a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp @@ -745,6 +745,10 @@ return internal_syscall(SYSCALL(getppid)); } +uptr internal_getcpu() { + return internal_syscall(SYSCALL(getcpu)); +} + int internal_dlinfo(void *handle, int request, void *p) { #if SANITIZER_FREEBSD return dlinfo(handle, request, p); diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.h b/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.h --- a/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.h @@ -41,6 +41,7 @@ void StackDepotLockAll(); void StackDepotUnlockAll(); +void StackDepotPrintAll(); // Instantiating this class creates a snapshot of StackDepot which can be // efficiently queried with StackDepotGet(). You can use it concurrently with diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.cpp --- a/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.cpp @@ -115,6 +115,12 @@ theDepot.UnlockAll(); } +void StackDepotPrintAll() { +#if !SANITIZER_GO + theDepot.PrintAll(); +#endif +} + bool StackDepotReverseMap::IdDescPair::IdComparator( const StackDepotReverseMap::IdDescPair &a, const StackDepotReverseMap::IdDescPair &b) { diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stackdepotbase.h b/compiler-rt/lib/sanitizer_common/sanitizer_stackdepotbase.h --- a/compiler-rt/lib/sanitizer_common/sanitizer_stackdepotbase.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_stackdepotbase.h @@ -13,6 +13,7 @@ #ifndef SANITIZER_STACKDEPOTBASE_H #define SANITIZER_STACKDEPOTBASE_H +#include #include "sanitizer_internal_defs.h" #include "sanitizer_mutex.h" #include "sanitizer_atomic.h" @@ -34,6 +35,7 @@ void LockAll(); void UnlockAll(); + void PrintAll(); private: static Node *find(Node *s, args_type args, u32 hash); @@ -172,6 +174,19 @@ } } +template +void StackDepotBase::PrintAll() { + for (int i = 0; i < kTabSize; ++i) { + atomic_uintptr_t *p = &tab[i]; + uptr v = atomic_load(p, memory_order_relaxed); + Node *s = (Node *)(v & ~1UL); + for (; s; s = s->link) { + Printf("Stack for id %u:\n", s->id); + s->load().Print(); + } + } +} + } // namespace __sanitizer #endif // SANITIZER_STACKDEPOTBASE_H diff --git a/compiler-rt/test/CMakeLists.txt b/compiler-rt/test/CMakeLists.txt --- a/compiler-rt/test/CMakeLists.txt +++ b/compiler-rt/test/CMakeLists.txt @@ -68,6 +68,9 @@ if(COMPILER_RT_BUILD_PROFILE AND COMPILER_RT_HAS_PROFILE) compiler_rt_test_runtime(profile) endif() + if(COMPILER_RT_BUILD_HEAPPROF) + compiler_rt_test_runtime(heapprof) + endif() if(COMPILER_RT_BUILD_XRAY) compiler_rt_test_runtime(xray) endif() diff --git a/compiler-rt/test/heapprof/CMakeLists.txt b/compiler-rt/test/heapprof/CMakeLists.txt new file mode 100644 --- /dev/null +++ b/compiler-rt/test/heapprof/CMakeLists.txt @@ -0,0 +1,40 @@ +set(HEAPPROF_LIT_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + +set(HEAPPROF_TESTSUITES) + +macro(get_bits_for_arch arch bits) + if (${arch} MATCHES "x86_64") + set(${bits} 64) + else() + message(FATAL_ERROR "Unexpected target architecture: ${arch}") + endif() +endmacro() + +set(HEAPPROF_TEST_DEPS ${SANITIZER_COMMON_LIT_TEST_DEPS}) +if(NOT COMPILER_RT_STANDALONE_BUILD) + list(APPEND HEAPPROF_TEST_DEPS heapprof) + if(COMPILER_RT_HAS_LLD AND TARGET lld) + list(APPEND HEAPPROF_TEST_DEPS lld) + endif() +endif() + +set(HEAPPROF_TEST_ARCH ${HEAPPROF_SUPPORTED_ARCH}) + +foreach(arch ${HEAPPROF_TEST_ARCH}) + set(HEAPPROF_TEST_TARGET_ARCH ${arch}) + string(TOLOWER "-${arch}-${OS_NAME}" HEAPPROF_TEST_CONFIG_SUFFIX) + get_bits_for_arch(${arch} HEAPPROF_TEST_BITS) + get_test_cc_for_arch(${arch} HEAPPROF_TEST_TARGET_CC HEAPPROF_TEST_TARGET_CFLAGS) + string(TOUPPER ${arch} ARCH_UPPER_CASE) + set(CONFIG_NAME ${ARCH_UPPER_CASE}${OS_NAME}Config) + configure_lit_site_cfg( + ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in + ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg.py + ) + list(APPEND HEAPPROF_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}) +endforeach() + +add_lit_testsuite(check-heapprof "Running the HeapProfiler tests" + ${HEAPPROF_TESTSUITES} + DEPENDS ${HEAPPROF_TEST_DEPS}) +set_target_properties(check-heapprof PROPERTIES FOLDER "Compiler-RT Misc") diff --git a/compiler-rt/test/heapprof/TestCases/atexit_stats.cpp b/compiler-rt/test/heapprof/TestCases/atexit_stats.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/test/heapprof/TestCases/atexit_stats.cpp @@ -0,0 +1,12 @@ +// Check atexit option. + +// RUN: %clangxx_heapprof -O0 %s -o %t +// RUN: %env_heapprof_opts=atexit=1 %run %t 2>&1 | FileCheck %s +// RUN: %env_heapprof_opts=atexit=0 %run %t 2>&1 | FileCheck %s --check-prefix=NOATEXIT + +// CHECK: HeapProfiler exit stats: +// NOATEXIT-NOT: HeapProfiler exit stats + +int main() { + return 0; +} diff --git a/compiler-rt/test/heapprof/TestCases/default_options.cpp b/compiler-rt/test/heapprof/TestCases/default_options.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/test/heapprof/TestCases/default_options.cpp @@ -0,0 +1,13 @@ +// RUN: %clangxx_heapprof -O2 %s -o %t +// RUN: %run %t 2>&1 | FileCheck %s + +const char *kHeapProfDefaultOptions = "verbosity=1 help=1"; + +extern "C" const char *__heapprof_default_options() { + // CHECK: Available flags for HeapProfiler: + return kHeapProfDefaultOptions; +} + +int main() { + return 0; +} diff --git a/compiler-rt/test/heapprof/TestCases/dump_process_map.cpp b/compiler-rt/test/heapprof/TestCases/dump_process_map.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/test/heapprof/TestCases/dump_process_map.cpp @@ -0,0 +1,14 @@ +// Check dump_process_map option. + +// RUN: %clangxx_heapprof -O0 %s -o %t +// RUN: %env_heapprof_opts=dump_process_map=1 %run %t 2>&1 | FileCheck %s +// RUN: %env_heapprof_opts=dump_process_map=0 %run %t 2>&1 | FileCheck %s --check-prefix=NOMAP + +// CHECK: Process memory map follows: +// CHECK: dump_process_map.cpp.tmp +// CHECK: End of process memory map. +// NOMAP-NOT: memory map + +int main() { + return 0; +} diff --git a/compiler-rt/test/heapprof/TestCases/free_hook_realloc.cpp b/compiler-rt/test/heapprof/TestCases/free_hook_realloc.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/test/heapprof/TestCases/free_hook_realloc.cpp @@ -0,0 +1,34 @@ +// Check that free hook doesn't conflict with Realloc. +// RUN: %clangxx_heapprof -O2 %s -o %t +// RUN: %run %t 2>&1 | FileCheck %s + +#include +#include +#include + +static void *glob_ptr; + +extern "C" { +void __sanitizer_free_hook(const volatile void *ptr) { + if (ptr == glob_ptr) { + *(int *)ptr = 0; + write(1, "FreeHook\n", sizeof("FreeHook\n")); + } +} +} + +int main() { + int *x = (int *)malloc(100); + x[0] = 42; + glob_ptr = x; + int *y = (int *)realloc(x, 200); + // Verify that free hook was called and didn't spoil the memory. + if (y[0] != 42) { + _exit(1); + } + write(1, "Passed\n", sizeof("Passed\n")); + free(y); + // CHECK: FreeHook + // CHECK: Passed + return 0; +} diff --git a/compiler-rt/test/heapprof/TestCases/function-sections-are-bad.cpp b/compiler-rt/test/heapprof/TestCases/function-sections-are-bad.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/test/heapprof/TestCases/function-sections-are-bad.cpp @@ -0,0 +1,39 @@ +// Check that --gc-sections does not throw away (or localize) parts of sanitizer +// interface. +// RUN: %clang_heapprof %s -Wl,--gc-sections -ldl -o %t +// RUN: %clang_heapprof %s -DBUILD_SO -fPIC -o %t-so.so -shared +// RUN: %run %t 2>&1 + +#ifndef BUILD_SO +#include +#include +#include +#include + +int main(int argc, char *argv[]) { + char path[4096]; + snprintf(path, sizeof(path), "%s-so.so", argv[0]); + + void *handle = dlopen(path, RTLD_LAZY); + if (!handle) fprintf(stderr, "%s\n", dlerror()); + assert(handle != 0); + + typedef void (*F)(); + F f = (F)dlsym(handle, "call_rtl_from_dso"); + printf("%s\n", dlerror()); + assert(dlerror() == 0); + f(); + + dlclose(handle); + return 0; +} + +#else // BUILD_SO + +#include +extern "C" void call_rtl_from_dso() { + volatile int32_t x; + volatile int32_t y = __sanitizer_unaligned_load32((void *)&x); +} + +#endif // BUILD_SO diff --git a/compiler-rt/test/heapprof/TestCases/heap_info_cache_entries.cpp b/compiler-rt/test/heapprof/TestCases/heap_info_cache_entries.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/test/heapprof/TestCases/heap_info_cache_entries.cpp @@ -0,0 +1,11 @@ +// Check heap_info_cache_entries option. + +// RUN: %clangxx_heapprof -O0 %s -o %t +// RUN: %env_heapprof_opts=heap_info_cache_entries=15:print_heap_info_cache_miss_rate=1:print_heap_info_cache_miss_rate_details=1 %run %t 2>&1 | FileCheck %s + +// CHECK: Set 14 miss rate: 0 / {{.*}} = 0.00% +// CHECK-NOT: Set + +int main() { + return 0; +} diff --git a/compiler-rt/test/heapprof/TestCases/heapprof_options-help.cpp b/compiler-rt/test/heapprof/TestCases/heapprof_options-help.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/test/heapprof/TestCases/heapprof_options-help.cpp @@ -0,0 +1,9 @@ +// RUN: %clangxx_heapprof -O0 %s -o %t +// RUN: %env_heapprof_opts=help=1 %run %t 2>&1 | FileCheck %s + +int main() { +} + +// CHECK: Available flags for HeapProfiler: +// CHECK-DAG: replace_str +// CHECK-DAG: print_stats diff --git a/compiler-rt/test/heapprof/TestCases/interface_test.cpp b/compiler-rt/test/heapprof/TestCases/interface_test.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/test/heapprof/TestCases/interface_test.cpp @@ -0,0 +1,10 @@ +// Check that user may include HeapProf interface header. +// RUN: %clang_heapprof %s -o %t && %run %t +// RUN: %clang_heapprof -x c %s -o %t && %run %t +// RUN: %clang %s -pie -o %t && %run %t +// RUN: %clang -x c %s -pie -o %t && %run %t +#include + +int main() { + return 0; +} diff --git a/compiler-rt/test/heapprof/TestCases/log_path_test.cpp b/compiler-rt/test/heapprof/TestCases/log_path_test.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/test/heapprof/TestCases/log_path_test.cpp @@ -0,0 +1,34 @@ +// The for loop in the backticks below requires bash. +// REQUIRES: shell +// +// RUN: %clangxx_heapprof %s -o %t + +// Regular run. +// RUN: %run %t 2> %t.out +// RUN: FileCheck %s --check-prefix=CHECK-GOOD < %t.out + +// Good log_path. +// RUN: rm -f %t.log.* +// RUN: %env_heapprof_opts=log_path=%t.log %run %t 2> %t.out +// RUN: FileCheck %s --check-prefix=CHECK-GOOD < %t.log.* + +// Invalid log_path. +// RUN: %env_heapprof_opts=log_path=/dev/null/INVALID not %run %t 2> %t.out +// RUN: FileCheck %s --check-prefix=CHECK-INVALID < %t.out + +// Too long log_path. +// RUN: %env_heapprof_opts=log_path=`for((i=0;i<10000;i++)); do echo -n $i; done` \ +// RUN: not %run %t 2> %t.out +// RUN: FileCheck %s --check-prefix=CHECK-LONG < %t.out + +#include +#include +int main(int argc, char **argv) { + char *x = (char *)malloc(10); + memset(x, 0, 10); + free(x); + return 0; +} +// CHECK-GOOD: Heap allocation stack id +// CHECK-INVALID: ERROR: Can't open file: /dev/null/INVALID +// CHECK-LONG: ERROR: Path is too long: 01234 diff --git a/compiler-rt/test/heapprof/TestCases/malloc-size-too-big.cpp b/compiler-rt/test/heapprof/TestCases/malloc-size-too-big.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/test/heapprof/TestCases/malloc-size-too-big.cpp @@ -0,0 +1,25 @@ +// RUN: %clangxx_heapprof -O0 %s -o %t +// RUN: %env_heapprof_opts=allocator_may_return_null=0 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-SUMMARY +// RUN: %env_heapprof_opts=allocator_may_return_null=1 %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-NULL +// Test print_summary +// RUN: %env_heapprof_opts=allocator_may_return_null=0:print_summary=0 not %run %t 2>&1 | FileCheck %s +// Test print_cmdline +// RUN: %env_heapprof_opts=allocator_may_return_null=0:print_cmdline=1 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-SUMMARY --check-prefix=CHECK-CMDLINE + +#include +#include + +static const size_t kMaxAllowedMallocSizePlusOne = (1ULL << 40) + 1; +int main() { + void *p = malloc(kMaxAllowedMallocSizePlusOne); + // CHECK: {{ERROR: HeapProfiler: requested allocation size .* \(.* after adjustments for alignment, headers etc\.\) exceeds maximum supported size}} + // CHECK: {{#0 0x.* in .*malloc}} + // CHECK: {{#1 0x.* in main .*malloc-size-too-big.cpp:}}[[@LINE-3]] + // CHECK-SUMMARY: SUMMARY: HeapProfiler: allocation-size-too-big + // CHECK-CMDLINE: Command: {{.*}}malloc-size-too-big.cpp.tmp + + printf("malloc returned: %zu\n", (size_t)p); + // CHECK-NULL: malloc returned: 0 + + return 0; +} diff --git a/compiler-rt/test/heapprof/TestCases/on_error_callback.cpp b/compiler-rt/test/heapprof/TestCases/on_error_callback.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/test/heapprof/TestCases/on_error_callback.cpp @@ -0,0 +1,17 @@ +// RUN: %clangxx_heapprof -O2 %s -o %t && not %run %t 2>&1 | FileCheck %s + +#include +#include + +extern "C" void __heapprof_on_error() { + fprintf(stderr, "__heapprof_on_error called\n"); + fflush(stderr); +} + +static const size_t kMaxAllowedMallocSizePlusOne = (1ULL << 40) + 1; +int main() { + void *p = malloc(kMaxAllowedMallocSizePlusOne); + // CHECK: __heapprof_on_error called + printf("malloc returned: %zu\n", (size_t)p); + return 0; +} diff --git a/compiler-rt/test/heapprof/TestCases/print_miss_rate.cpp b/compiler-rt/test/heapprof/TestCases/print_miss_rate.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/test/heapprof/TestCases/print_miss_rate.cpp @@ -0,0 +1,14 @@ +// Check print_heap_info_cache_miss_rate and +// print_heap_info_cache_miss_rate_details options. + +// RUN: %clangxx_heapprof -O0 %s -o %t +// RUN: %env_heapprof_opts=print_heap_info_cache_miss_rate=1 %run %t 2>&1 | FileCheck %s +// RUN: %env_heapprof_opts=print_heap_info_cache_miss_rate=1:print_heap_info_cache_miss_rate_details=1 %run %t 2>&1 | FileCheck %s --check-prefix=DETAILS + +// CHECK: Overall miss rate: 0 / {{.*}} = 0.00% +// DETAILS: Set 0 miss rate: 0 / {{.*}} = 0.00% +// DETAILS: Set 16380 miss rate: 0 / {{.*}} = 0.00% + +int main() { + return 0; +} diff --git a/compiler-rt/test/heapprof/TestCases/realloc.cpp b/compiler-rt/test/heapprof/TestCases/realloc.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/test/heapprof/TestCases/realloc.cpp @@ -0,0 +1,21 @@ +// RUN: %clangxx_heapprof -O0 %s -o %t +// Default is true (free on realloc to 0 size) +// RUN: %run %t 2>&1 | FileCheck %s +// RUN: %env_heapprof_opts=allocator_frees_and_returns_null_on_realloc_zero=true %run %t 2>&1 | FileCheck %s +// RUN: %env_heapprof_opts=allocator_frees_and_returns_null_on_realloc_zero=false %run %t 2>&1 | FileCheck %s --check-prefix=NO-FREE + +#include +#include + +int main() { + void *p = malloc(42); + p = realloc(p, 0); + if (p) { + // NO-FREE: Allocated something on realloc(p, 0) + fprintf(stderr, "Allocated something on realloc(p, 0)\n"); + } else { + // CHECK: realloc(p, 0) returned nullptr + fprintf(stderr, "realloc(p, 0) returned nullptr\n"); + } + free(p); +} diff --git a/compiler-rt/test/heapprof/TestCases/sleep_after_init.c b/compiler-rt/test/heapprof/TestCases/sleep_after_init.c new file mode 100644 --- /dev/null +++ b/compiler-rt/test/heapprof/TestCases/sleep_after_init.c @@ -0,0 +1,10 @@ +// RUN: %clang_heapprof -O2 %s -o %t +// RUN: %env_heapprof_opts=sleep_after_init=1 %run %t 2>&1 | FileCheck %s + +#include +int main() { + // CHECK: Sleeping for 1 second + char *x = (char *)malloc(10 * sizeof(char)); + free(x); + return 0; +} diff --git a/compiler-rt/test/heapprof/TestCases/sleep_before_dying.c b/compiler-rt/test/heapprof/TestCases/sleep_before_dying.c new file mode 100644 --- /dev/null +++ b/compiler-rt/test/heapprof/TestCases/sleep_before_dying.c @@ -0,0 +1,11 @@ +// RUN: %clang_heapprof -O2 %s -o %t +// RUN: %env_heapprof_opts=sleep_before_dying=1 not %run %t 2>&1 | FileCheck %s + +#include +#include +static const size_t kMaxAllowedMallocSizePlusOne = (1ULL << 40) + 1; +int main() { + void *p = malloc(kMaxAllowedMallocSizePlusOne); + // CHECK: Sleeping for 1 second + printf("malloc returned: %zu\n", (size_t)p); +} diff --git a/compiler-rt/test/heapprof/TestCases/test_malloc_load_store.c b/compiler-rt/test/heapprof/TestCases/test_malloc_load_store.c new file mode 100644 --- /dev/null +++ b/compiler-rt/test/heapprof/TestCases/test_malloc_load_store.c @@ -0,0 +1,38 @@ +// Check profile with a single malloc call and set of loads and stores. Ensures +// we get the same profile regardless of whether the memory is deallocated +// before exit. + +// RUN: %clangxx_heapprof -O0 %s -o %t +// RUN: %env_heapprof_opts= %run %t 2>&1 | FileCheck %s + +// RUN: %clangxx_heapprof -DFREE -O0 %s -o %t +// RUN: %env_heapprof_opts= %run %t 2>&1 | FileCheck %s + +// This is actually: +// Heap allocation stack id = STACKID +// alloc_count 1, size (ave/min/max) 40.00 / 40 / 40 +// but we need to look for them in the same CHECK to get the correct STACKID. +// CHECK: Heap allocation stack id = [[STACKID:[0-9]+]]{{[[:space:]].*}}alloc_count 1, size (ave/min/max) 40.00 / 40 / 40 +// CHECK-NEXT: access_count (ave/min/max): 20.00 / 20 / 20 +// CHECK-NEXT: lifetime (ave/min/max): [[AVELIFETIME:[0-9]+]].00 / [[AVELIFETIME]] / [[AVELIFETIME]] +// CHECK-NEXT: num migrated: 0, num lifetime overlaps: 0, num same alloc cpu: 0, num same dealloc_cpu: 0 +// CHECK: Stack for id [[STACKID]]: +// CHECK-NEXT: #0 {{.*}} in malloc +// CHECK-NEXT: #1 {{.*}} in main {{.*}}:[[@LINE+6]] + +#include +#include + +int main() { + int *p = (int *)malloc(10 * sizeof(int)); + for (int i = 0; i < 10; i++) + p[i] = i; + int j = 0; + for (int i = 0; i < 10; i++) + j += p[i]; +#ifdef FREE + free(p); +#endif + + return 0; +} diff --git a/compiler-rt/test/heapprof/TestCases/test_memintrin.cpp b/compiler-rt/test/heapprof/TestCases/test_memintrin.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/test/heapprof/TestCases/test_memintrin.cpp @@ -0,0 +1,49 @@ +// Check profile with calls to memory intrinsics. + +// RUN: %clangxx_heapprof -O0 %s -o %t +// RUN: %run %t 2>&1 | FileCheck %s + +// This is actually: +// Heap allocation stack id = STACKIDP +// alloc_count 1, size (ave/min/max) 40.00 / 40 / 40 +// access_count (ave/min/max): 3.00 / 3 / 3 +// but we need to look for them in the same CHECK to get the correct STACKIDP. +// CHECK-DAG: Heap allocation stack id = [[STACKIDP:[0-9]+]]{{[[:space:]].*}} alloc_count 1, size (ave/min/max) 40.00 / 40 / 40{{[[:space:]].*}} access_count (ave/min/max): 3.00 / 3 / 3 +// +// This is actually: +// Heap allocation stack id = STACKIDQ +// alloc_count 1, size (ave/min/max) 20.00 / 20 / 20 +// access_count (ave/min/max): 2.00 / 2 / 2 +// but we need to look for them in the same CHECK to get the correct STACKIDQ. +// CHECK-DAG: Heap allocation stack id = [[STACKIDQ:[0-9]+]]{{[[:space:]].*}} alloc_count 1, size (ave/min/max) 20.00 / 20 / 20{{[[:space:]].*}} access_count (ave/min/max): 2.00 / 2 / 2 + +#include +#include +#include + +int main() { + // This is actually: + // Stack for id STACKIDP: + // #0 {{.*}} in operator new + // #1 {{.*}} in main {{.*}}:@LINE+1 + // but we need to look for them in the same CHECK-DAG. + // CHECK-DAG: Stack for id [[STACKIDP]]:{{[[:space:]].*}} #0 {{.*}} in operator new{{.*[[:space:]].*}} #1 {{.*}} in main {{.*}}:[[@LINE+1]] + int *p = new int[10]; + + // This is actually: + // Stack for id STACKIDQ: + // #0 {{.*}} in operator new + // #1 {{.*}} in main {{.*}}:@LINE+1 + // but we need to look for them in the same CHECK-DAG. + // CHECK-DAG: Stack for id [[STACKIDQ]]:{{[[:space:]].*}} #0 {{.*}} in operator new{{.*[[:space:]].*}} #1 {{.*}} in main {{.*}}:[[@LINE+1]] + int *q = new int[5]; + + memset(p, 1, 10); + memcpy(q, p, 5); + int x = memcmp(p, q, 5); + + delete p; + delete q; + + return x; +} diff --git a/compiler-rt/test/heapprof/TestCases/test_new_load_store.cpp b/compiler-rt/test/heapprof/TestCases/test_new_load_store.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/test/heapprof/TestCases/test_new_load_store.cpp @@ -0,0 +1,42 @@ +// Check profile with a single new call and set of loads and stores. Ensures +// we get the same profile regardless of whether the memory is deallocated +// before exit. + +// RUN: %clangxx_heapprof -O0 %s -o %t +// RUN: %env_heapprof_opts= %run %t 2>&1 | FileCheck %s + +// RUN: %clangxx_heapprof -DFREE -O0 %s -o %t +// RUN: %env_heapprof_opts= %run %t 2>&1 | FileCheck %s + +// Try again with callbacks instead of inline sequences +// RUN: %clangxx_heapprof -mllvm -heapprof-use-callbacks -O0 %s -o %t +// RUN: %env_heapprof_opts= %run %t 2>&1 | FileCheck %s + +// This is actually: +// Heap allocation stack id = STACKID +// alloc_count 1, size (ave/min/max) 40.00 / 40 / 40 +// but we need to look for them in the same CHECK to get the correct STACKID. +// CHECK: Heap allocation stack id = [[STACKID:[0-9]+]]{{[[:space:]].*}}alloc_count 1, size (ave/min/max) 40.00 / 40 / 40 +// CHECK-NEXT: access_count (ave/min/max): 20.00 / 20 / 20 +// CHECK-NEXT: lifetime (ave/min/max): [[AVELIFETIME:[0-9]+]].00 / [[AVELIFETIME]] / [[AVELIFETIME]] +// CHECK-NEXT: num migrated: 0, num lifetime overlaps: 0, num same alloc cpu: 0, num same dealloc_cpu: 0 +// CHECK: Stack for id [[STACKID]]: +// CHECK-NEXT: #0 {{.*}} in operator new +// CHECK-NEXT: #1 {{.*}} in main {{.*}}:[[@LINE+6]] + +#include +#include + +int main() { + int *p = new int[10]; + for (int i = 0; i < 10; i++) + p[i] = i; + int j = 0; + for (int i = 0; i < 10; i++) + j += p[i]; +#ifdef FREE + delete p; +#endif + + return 0; +} diff --git a/compiler-rt/test/heapprof/TestCases/test_terse.cpp b/compiler-rt/test/heapprof/TestCases/test_terse.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/test/heapprof/TestCases/test_terse.cpp @@ -0,0 +1,31 @@ +// Check terse format profile with a single malloc call and set of loads and +// stores. Ensures we get the same profile regardless of whether the memory is +// deallocated before exit. + +// RUN: %clangxx_heapprof -O0 %s -o %t +// RUN: %env_heapprof_opts=print_terse=1 %run %t 2>&1 | FileCheck %s + +// RUN: %clangxx_heapprof -DFREE -O0 %s -o %t +// RUN: %env_heapprof_opts=print_terse=1 %run %t 2>&1 | FileCheck %s + +// CHECK: HIB:[[STACKID:[0-9]+]]/1/40.00/40/40/20.00/20/20/[[AVELIFETIME:[0-9]+]].00/[[AVELIFETIME]]/[[AVELIFETIME]]/0/0/0/0 +// CHECK: Stack for id [[STACKID]]: +// CHECK-NEXT: #0 {{.*}} in operator new +// CHECK-NEXT: #1 {{.*}} in main {{.*}}:[[@LINE+6]] + +#include +#include + +int main() { + int *p = new int[10]; + for (int i = 0; i < 10; i++) + p[i] = i; + int j = 0; + for (int i = 0; i < 10; i++) + j += p[i]; +#ifdef FREE + delete p; +#endif + + return 0; +} diff --git a/compiler-rt/test/heapprof/lit.cfg.py b/compiler-rt/test/heapprof/lit.cfg.py new file mode 100644 --- /dev/null +++ b/compiler-rt/test/heapprof/lit.cfg.py @@ -0,0 +1,89 @@ +# -*- Python -*- + +import os +import platform +import re + +import lit.formats + +# Get shlex.quote if available (added in 3.3), and fall back to pipes.quote if +# it's not available. +try: + import shlex + sh_quote = shlex.quote +except: + import pipes + sh_quote = pipes.quote + +def get_required_attr(config, attr_name): + attr_value = getattr(config, attr_name, None) + if attr_value == None: + lit_config.fatal( + "No attribute %r in test configuration! You may need to run " + "tests from your build directory or add this attribute " + "to lit.site.cfg.py " % attr_name) + return attr_value + +# Setup config name. +config.name = 'HeapProfiler' + config.name_suffix + +# Platform-specific default HEAPPROF_OPTIONS for lit tests. +default_heapprof_opts = list(config.default_sanitizer_opts) + +default_heapprof_opts_str = ':'.join(default_heapprof_opts) +if default_heapprof_opts_str: + config.environment['HEAPPROF_OPTIONS'] = default_heapprof_opts_str + default_heapprof_opts_str += ':' +config.substitutions.append(('%env_heapprof_opts=', + 'env HEAPPROF_OPTIONS=' + default_heapprof_opts_str)) + +# Setup source root. +config.test_source_root = os.path.dirname(__file__) + +libdl_flag = "-ldl" + +# Setup default compiler flags used with -fmemprof option. +# FIXME: Review the set of required flags and check if it can be reduced. +target_cflags = [get_required_attr(config, "target_cflags")] +target_cxxflags = config.cxx_mode_flags + target_cflags +clang_heapprof_static_cflags = (["-fmemprof", + "-mno-omit-leaf-frame-pointer", + "-fno-omit-frame-pointer", + "-fno-optimize-sibling-calls"] + + config.debug_info_flags + target_cflags) +clang_heapprof_static_cxxflags = config.cxx_mode_flags + clang_heapprof_static_cflags + +config.available_features.add("heapprof-static-runtime") +clang_heapprof_cflags = clang_heapprof_static_cflags +clang_heapprof_cxxflags = clang_heapprof_static_cxxflags + +def build_invocation(compile_flags): + return " " + " ".join([config.clang] + compile_flags) + " " + +config.substitutions.append( ("%clang ", build_invocation(target_cflags)) ) +config.substitutions.append( ("%clangxx ", build_invocation(target_cxxflags)) ) +config.substitutions.append( ("%clang_heapprof ", build_invocation(clang_heapprof_cflags)) ) +config.substitutions.append( ("%clangxx_heapprof ", build_invocation(clang_heapprof_cxxflags)) ) + +# Some tests uses C++11 features such as lambdas and need to pass -std=c++11. +config.substitutions.append(("%stdcxx11 ", "-std=c++11 ")) + +config.substitutions.append( ("%libdl", libdl_flag) ) + +config.available_features.add("heapprof-" + config.bits + "-bits") + +config.available_features.add('fast-unwinder-works') + +# Default test suffixes. +config.suffixes = ['.c', '.cpp'] + +config.substitutions.append(('%fPIC', '-fPIC')) +config.substitutions.append(('%fPIE', '-fPIE')) +config.substitutions.append(('%pie', '-pie')) + +# Only run the tests on supported OSs. +if config.host_os not in ['Linux']: + config.unsupported = True + +if not config.parallelism_group: + config.parallelism_group = 'shadow-memory' diff --git a/compiler-rt/test/heapprof/lit.site.cfg.py.in b/compiler-rt/test/heapprof/lit.site.cfg.py.in new file mode 100644 --- /dev/null +++ b/compiler-rt/test/heapprof/lit.site.cfg.py.in @@ -0,0 +1,14 @@ +@LIT_SITE_CFG_IN_HEADER@ + +# Tool-specific config options. +config.name_suffix = "@HEAPPROF_TEST_CONFIG_SUFFIX@" +config.target_cflags = "@HEAPPROF_TEST_TARGET_CFLAGS@" +config.clang = "@HEAPPROF_TEST_TARGET_CC@" +config.bits = "@HEAPPROF_TEST_BITS@" +config.target_arch = "@HEAPPROF_TEST_TARGET_ARCH@" + +# Load common config for all compiler-rt lit tests. +lit_config.load_config(config, "@COMPILER_RT_BINARY_DIR@/test/lit.common.configured") + +# Load tool-specific config that would do the real work. +lit_config.load_config(config, "@HEAPPROF_LIT_SOURCE_DIR@/lit.cfg.py") diff --git a/compiler-rt/test/lit.common.cfg.py b/compiler-rt/test/lit.common.cfg.py --- a/compiler-rt/test/lit.common.cfg.py +++ b/compiler-rt/test/lit.common.cfg.py @@ -57,6 +57,8 @@ # If needed, add cflag for shadow scale. if config.asan_shadow_scale != '': config.target_cflags += " -mllvm -asan-mapping-scale=" + config.asan_shadow_scale +if config.heapprof_shadow_scale != '': + config.target_cflags += " -mllvm -heapprof-mapping-scale=" + config.heapprof_shadow_scale # BFD linker in 64-bit android toolchains fails to find libc++_shared.so, which # is a transitive shared library dependency (via asan runtime). @@ -542,6 +544,11 @@ else: config.available_features.add("shadow-scale-3") +if config.heapprof_shadow_scale: + config.available_features.add("heapprof-shadow-scale-%s" % config.heapprof_shadow_scale) +else: + config.available_features.add("heapprof-shadow-scale-3") + if config.expensive_checks: config.available_features.add("expensive_checks") diff --git a/compiler-rt/test/lit.common.configured.in b/compiler-rt/test/lit.common.configured.in --- a/compiler-rt/test/lit.common.configured.in +++ b/compiler-rt/test/lit.common.configured.in @@ -29,6 +29,7 @@ set_default("compiler_rt_libdir", "@COMPILER_RT_RESOLVED_LIBRARY_OUTPUT_DIR@") set_default("emulator", "@COMPILER_RT_EMULATOR@") set_default("asan_shadow_scale", "@COMPILER_RT_ASAN_SHADOW_SCALE@") +set_default("heapprof_shadow_scale", "@COMPILER_RT_HEAPPROF_SHADOW_SCALE@") set_default("apple_platform", "osx") set_default("apple_platform_min_deployment_target_flag", "-mmacosx-version-min") set_default("sanitizer_can_use_cxxabi", @SANITIZER_CAN_USE_CXXABI_PYBOOL@)