Index: compiler-rt/lib/scudo/standalone/allocator_config.h =================================================================== --- compiler-rt/lib/scudo/standalone/allocator_config.h +++ compiler-rt/lib/scudo/standalone/allocator_config.h @@ -79,7 +79,11 @@ typedef MapAllocatorCache SecondaryCache; static const u32 SecondaryCacheEntriesArraySize = 32U; +#ifdef _MSC_VER + static const u32 SecondaryCacheQuarantineSize = 1U; +#else static const u32 SecondaryCacheQuarantineSize = 0U; +#endif static const u32 SecondaryCacheDefaultMaxEntriesCount = 32U; static const uptr SecondaryCacheDefaultMaxEntrySize = 1UL << 19; static const s32 SecondaryCacheMinReleaseToOsIntervalMs = INT32_MIN; @@ -87,6 +91,7 @@ template using TSDRegistryT = TSDRegistryExT; // Exclusive }; + struct AndroidConfig { using SizeClassMap = AndroidSizeClassMap; static const bool MaySupportMemoryTagging = true; @@ -149,6 +154,37 @@ using TSDRegistryT = TSDRegistrySharedT; // Shared, max 2 TSDs. }; +struct WindowsConfig { + using SizeClassMap = DefaultSizeClassMap; + static const bool MaySupportMemoryTagging = false; + +#if SCUDO_CAN_USE_PRIMARY64 + typedef SizeClassAllocator64 Primary; + static const uptr PrimaryRegionSizeLog = 33U; + typedef uptr PrimaryCompactPtrT; + static const uptr PrimaryCompactPtrScale = 0; + static const bool PrimaryEnableRandomOffset = true; + static const uptr PrimaryMapSizeIncrement = 1UL << 18; +#else + typedef SizeClassAllocator32 Primary; + static const uptr PrimaryRegionSizeLog = 19U; + typedef uptr PrimaryCompactPtrT; +#endif + static const s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN; + static const s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX; + + typedef MapAllocatorCache SecondaryCache; + static const u32 SecondaryCacheEntriesArraySize = 32U; + static const u32 SecondaryCacheQuarantineSize = 2U; + static const u32 SecondaryCacheDefaultMaxEntriesCount = 32U; + static const uptr SecondaryCacheDefaultMaxEntrySize = PTRBIT << 19; + static const s32 SecondaryCacheMinReleaseToOsIntervalMs = INT32_MIN; + static const s32 SecondaryCacheMaxReleaseToOsIntervalMs = INT32_MAX; + + template + using TSDRegistryT = TSDRegistrySharedT; // Shared, max 16 TSDs. +}; + #if SCUDO_CAN_USE_PRIMARY64 struct FuchsiaConfig { using SizeClassMap = FuchsiaSizeClassMap; @@ -191,6 +227,8 @@ #if SCUDO_ANDROID typedef AndroidConfig Config; +#elif SCUDO_WINDOWS +typedef WindowsConfig Config; #elif SCUDO_FUCHSIA typedef FuchsiaConfig Config; #elif SCUDO_TRUSTY Index: compiler-rt/lib/scudo/standalone/atomic_helpers.h =================================================================== --- compiler-rt/lib/scudo/standalone/atomic_helpers.h +++ compiler-rt/lib/scudo/standalone/atomic_helpers.h @@ -11,6 +11,10 @@ #include "internal_defs.h" +#ifdef _WIN32 +#include +#include +#endif namespace scudo { enum memory_order { @@ -21,86 +25,151 @@ memory_order_acq_rel = 4, memory_order_seq_cst = 5 }; +#ifndef _MSC_VER static_assert(memory_order_relaxed == __ATOMIC_RELAXED, ""); static_assert(memory_order_consume == __ATOMIC_CONSUME, ""); static_assert(memory_order_acquire == __ATOMIC_ACQUIRE, ""); static_assert(memory_order_release == __ATOMIC_RELEASE, ""); static_assert(memory_order_acq_rel == __ATOMIC_ACQ_REL, ""); static_assert(memory_order_seq_cst == __ATOMIC_SEQ_CST, ""); +#endif struct atomic_u8 { typedef u8 Type; +#ifdef _MSC_VER + volatile std::atomic_uint8_t ValDoNotUse; +#else volatile Type ValDoNotUse; +#endif }; struct atomic_u16 { typedef u16 Type; +#ifdef _MSC_VER + volatile std::atomic_uint16_t ValDoNotUse; +#else volatile Type ValDoNotUse; +#endif }; struct atomic_s32 { typedef s32 Type; +#ifdef _MSC_VER + volatile std::atomic_int32_t ValDoNotUse; +#else volatile Type ValDoNotUse; +#endif }; struct atomic_u32 { typedef u32 Type; +#ifdef _MSC_VER + volatile std::atomic_uint32_t ValDoNotUse; +#else volatile Type ValDoNotUse; +#endif }; struct atomic_u64 { typedef u64 Type; // On 32-bit platforms u64 is not necessarily aligned on 8 bytes. +#ifdef _MSC_VER + volatile std::atomic_uint64_t ValDoNotUse; +#else alignas(8) volatile Type ValDoNotUse; +#endif }; struct atomic_uptr { typedef uptr Type; +#ifdef _MSC_VER + volatile std::atomic_uintptr_t ValDoNotUse; +#else volatile Type ValDoNotUse; +#endif }; template inline typename T::Type atomic_load(const volatile T *A, memory_order MO) { DCHECK(!(reinterpret_cast(A) % sizeof(*A))); +#ifdef _MSC_VER + return std::atomic_load_explicit(&A->ValDoNotUse, static_cast(MO)); +#else typename T::Type V; __atomic_load(&A->ValDoNotUse, &V, MO); return V; +#endif } template inline void atomic_store(volatile T *A, typename T::Type V, memory_order MO) { DCHECK(!(reinterpret_cast(A) % sizeof(*A))); +#ifdef _MSC_VER + std::atomic_store_explicit(&A->ValDoNotUse, V, static_cast(MO)); +#else __atomic_store(&A->ValDoNotUse, &V, MO); +#endif } -inline void atomic_thread_fence(memory_order) { __sync_synchronize(); } +inline void atomic_thread_fence(memory_order) { +#ifdef _AMD64_ + __faststorefence(); +#elif defined(_IA64_) + __mf() +#elif defined(_MSC_VER) + MemoryBarrier(); +#else + __sync_synchronize(); +#endif +} +#ifdef _WIN32 +inline void __atomic_signal_fence(memory_order order) { + std::atomic_signal_fence(static_cast(order)); +} +#endif template inline typename T::Type atomic_fetch_add(volatile T *A, typename T::Type V, memory_order MO) { DCHECK(!(reinterpret_cast(A) % sizeof(*A))); +#ifdef _MSC_VER + return std::atomic_fetch_add_explicit(&A->ValDoNotUse, V, static_cast(MO)); +#else return __atomic_fetch_add(&A->ValDoNotUse, V, MO); +#endif } template inline typename T::Type atomic_fetch_sub(volatile T *A, typename T::Type V, memory_order MO) { DCHECK(!(reinterpret_cast(A) % sizeof(*A))); +#ifdef _MSC_VER + return std::atomic_fetch_sub_explicit(&A->ValDoNotUse, V, static_cast(MO)); +#else return __atomic_fetch_sub(&A->ValDoNotUse, V, MO); +#endif } template inline typename T::Type atomic_fetch_and(volatile T *A, typename T::Type V, memory_order MO) { DCHECK(!(reinterpret_cast(A) % sizeof(*A))); +#ifdef _MSC_VER + return std::atomic_fetch_and(&A->ValDoNotUse, V); +#else return __atomic_fetch_and(&A->ValDoNotUse, V, MO); +#endif } template inline typename T::Type atomic_fetch_or(volatile T *A, typename T::Type V, memory_order MO) { DCHECK(!(reinterpret_cast(A) % sizeof(*A))); +#ifdef _MSC_VER + return std::atomic_fetch_or(&A->ValDoNotUse, V); +#else return __atomic_fetch_or(&A->ValDoNotUse, V, MO); +#endif } template @@ -108,7 +177,11 @@ memory_order MO) { DCHECK(!(reinterpret_cast(A) % sizeof(*A))); typename T::Type R; +#ifdef _MSC_VER + R = std::atomic_exchange_explicit(&A->ValDoNotUse, V, static_cast(MO)); +#else __atomic_exchange(&A->ValDoNotUse, &V, &R, MO); +#endif return R; } @@ -116,8 +189,12 @@ inline bool atomic_compare_exchange_strong(volatile T *A, typename T::Type *Cmp, typename T::Type Xchg, memory_order MO) { +#ifdef _MSC_VER + return std::atomic_compare_exchange_strong(&A->ValDoNotUse, Cmp, Xchg); +#else return __atomic_compare_exchange(&A->ValDoNotUse, Cmp, &Xchg, false, MO, - __ATOMIC_RELAXED); + memory_order_relaxed); +#endif } // Clutter-reducing helpers. Index: compiler-rt/lib/scudo/standalone/chunk.h =================================================================== --- compiler-rt/lib/scudo/standalone/chunk.h +++ compiler-rt/lib/scudo/standalone/chunk.h @@ -16,6 +16,10 @@ #include "common.h" #include "report.h" +#ifdef _WIN32 +#include +#endif + namespace scudo { extern Checksum HashAlgorithm; @@ -25,7 +29,12 @@ // as opposed to only for crc32_hw.cpp. This means that other hardware // specific instructions were likely emitted at other places, and as a result // there is no reason to not use it here. -#if defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) +#ifdef _WIN32 + u32 Crc = static_cast(_mm_crc32_u32(Seed, static_cast(Value))); + for (uptr I = 0; I < ArraySize; I++) + Crc = static_cast(_mm_crc32_u32(Crc, static_cast(Array[I]))); + return static_cast(Crc ^ (Crc >> 16)); +#elif defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) u32 Crc = static_cast(CRC32_INTRINSIC(Seed, Value)); for (uptr I = 0; I < ArraySize; I++) Crc = static_cast(CRC32_INTRINSIC(Crc, Array[I])); @@ -62,7 +71,18 @@ typedef u64 PackedHeader; // Update the 'Mask' constants to reflect changes in this structure. +#ifdef _MSC_VER +#pragma pack(push,1) +#endif struct UnpackedHeader { +#ifdef _MSC_VER + unsigned int ClassId : 8; + unsigned int State : 2; + unsigned int OriginOrWasZeroed : 2; + unsigned int SizeOrUnusedBytes : 20; + unsigned int Offset : 16; + unsigned int Checksum : 16; +#else uptr ClassId : 8; u8 State : 2; // Origin if State == Allocated, or WasZeroed otherwise. @@ -70,21 +90,25 @@ uptr SizeOrUnusedBytes : 20; uptr Offset : 16; uptr Checksum : 16; +#endif }; +#ifdef _MSC_VER +#pragma pack(pop) +#endif typedef atomic_u64 AtomicPackedHeader; static_assert(sizeof(UnpackedHeader) == sizeof(PackedHeader), ""); // Those constants are required to silence some -Werror=conversion errors when // assigning values to the related bitfield variables. -constexpr uptr ClassIdMask = (1UL << 8) - 1; +constexpr uptr ClassIdMask = (PTRBIT << 8) - 1; constexpr u8 StateMask = (1U << 2) - 1; constexpr u8 OriginMask = (1U << 2) - 1; -constexpr uptr SizeOrUnusedBytesMask = (1UL << 20) - 1; -constexpr uptr OffsetMask = (1UL << 16) - 1; -constexpr uptr ChecksumMask = (1UL << 16) - 1; +constexpr uptr SizeOrUnusedBytesMask = (PTRBIT << 20) - 1; +constexpr uptr OffsetMask = (PTRBIT << 16) - 1; +constexpr uptr ChecksumMask = (PTRBIT << 16) - 1; constexpr uptr getHeaderSize() { - return roundUpTo(sizeof(PackedHeader), 1U << SCUDO_MIN_ALIGNMENT_LOG); + return roundUpTo(sizeof(PackedHeader), PTRBIT << SCUDO_MIN_ALIGNMENT_LOG); } inline AtomicPackedHeader *getAtomicHeader(void *Ptr) { Index: compiler-rt/lib/scudo/standalone/combined.h =================================================================== --- compiler-rt/lib/scudo/standalone/combined.h +++ compiler-rt/lib/scudo/standalone/combined.h @@ -51,6 +51,11 @@ typedef typename Params::template TSDRegistryT TSDRegistryT; void callPostInitCallback() { +#ifdef _WIN32 + static pthread_once_t PostInitNonce; +#else + static pthread_once_t PostInitNonce = PTHREAD_ONCE_INIT; +#endif pthread_once(&PostInitNonce, PostInitCallback); } @@ -962,10 +967,10 @@ static const uptr MinAlignmentLog = SCUDO_MIN_ALIGNMENT_LOG; static const uptr MaxAlignmentLog = 24U; // 16 MB seems reasonable. - static const uptr MinAlignment = 1UL << MinAlignmentLog; - static const uptr MaxAlignment = 1UL << MaxAlignmentLog; + static const uptr MinAlignment = PTRBIT << MinAlignmentLog; + static const uptr MaxAlignment = PTRBIT << MaxAlignmentLog; static const uptr MaxAllowedMallocSize = - FIRST_32_SECOND_64(1UL << 31, 1ULL << 40); + FIRST_32_SECOND_64(PTRBIT << 31, PTRBIT << 40); static_assert(MinAlignment >= sizeof(Chunk::PackedHeader), "Minimal alignment must at least cover a chunk header."); @@ -985,19 +990,19 @@ static const sptr MemTagAllocationTraceIndex = -2; static const sptr MemTagAllocationTidIndex = -1; - u32 Cookie = 0; - u32 QuarantineMaxChunkSize = 0; + u32 Cookie MAYBE_INIT(=0); + u32 QuarantineMaxChunkSize MAYBE_INIT(= 0); GlobalStats Stats; PrimaryT Primary; SecondaryT Secondary; QuarantineT Quarantine; TSDRegistryT TSDRegistry; - pthread_once_t PostInitNonce = PTHREAD_ONCE_INIT; + pthread_once_t PostInitNonce MAYBE_INIT(= PTHREAD_ONCE_INIT); #ifdef GWP_ASAN_HOOKS gwp_asan::GuardedPoolAllocator GuardedAlloc; - uptr GuardedAllocSlotSize = 0; + uptr GuardedAllocSlotSize MAYBE_INIT(= 0); #endif // GWP_ASAN_HOOKS StackDepot Depot; @@ -1020,7 +1025,7 @@ #endif Entry Entries[NumEntries]; }; - AllocationRingBuffer RingBuffer = {}; + AllocationRingBuffer RingBuffer MAYBE_INIT(= {}); // The following might get optimized out by the compiler. NOINLINE void performSanityChecks() { @@ -1033,7 +1038,7 @@ // result, the maximum offset will be at most the maximum alignment for the // last size class minus the header size, in multiples of MinAlignment. Chunk::UnpackedHeader Header = {}; - const uptr MaxPrimaryAlignment = 1UL << getMostSignificantSetBitIndex( + const uptr MaxPrimaryAlignment = PTRBIT << getMostSignificantSetBitIndex( SizeClassMap::MaxSize - MinAlignment); const uptr MaxOffset = (MaxPrimaryAlignment - Chunk::getHeaderSize()) >> MinAlignmentLog; @@ -1105,7 +1110,7 @@ NewHeader.ClassId); // Exclude the previous tag so that immediate use after free is // detected 100% of the time. - setRandomTag(Ptr, Size, OddEvenMask | (1UL << PrevTag), &TaggedBegin, + setRandomTag(Ptr, Size, OddEvenMask | (PTRBIT << PrevTag), &TaggedBegin, &TaggedEnd); } } Index: compiler-rt/lib/scudo/standalone/common.h =================================================================== --- compiler-rt/lib/scudo/standalone/common.h +++ compiler-rt/lib/scudo/standalone/common.h @@ -11,8 +11,16 @@ #include "internal_defs.h" +#ifdef _MSC_VER +#include +#include +#define LIKELY(X) (X) +#define UNLIKELY(X) (X) +#endif + #include "fuchsia.h" #include "linux.h" +#include "wind.h" #include "trusty.h" #include @@ -51,9 +59,23 @@ inline bool isPowerOfTwo(uptr X) { return (X & (X - 1)) == 0; } +#ifdef _WIN32 +static inline int __builtin_clz(unsigned x) { + return (int)__lzcnt(x); +} + +static inline int __builtin_clzll(unsigned long long x) { + return (int)__lzcnt64(x); +} + +static inline int __builtin_clzl(unsigned long long x) { + return sizeof(x) == 8 ? __builtin_clzll(x) : __builtin_clz((uint32_t)x); +} +#endif + inline uptr getMostSignificantSetBitIndex(uptr X) { DCHECK_NE(X, 0U); - return SCUDO_WORDSIZE - 1U - static_cast(__builtin_clzl(X)); + return static_cast(SCUDO_WORDSIZE - 1U) - static_cast(__builtin_clzl(X)); } inline uptr roundUpToPowerOfTwo(uptr Size) { @@ -61,10 +83,28 @@ if (isPowerOfTwo(Size)) return Size; const uptr Up = getMostSignificantSetBitIndex(Size); - DCHECK_LT(Size, (1UL << (Up + 1))); - DCHECK_GT(Size, (1UL << Up)); - return 1UL << (Up + 1); + DCHECK_LT(Size, (PTRBIT << (Up + 1))); + DCHECK_GT(Size, (PTRBIT << Up)); + return PTRBIT << (Up + 1); +} + +#ifdef _WIN32 +static inline int __builtin_ctz(unsigned x) { + unsigned long ret; + _BitScanForward(&ret, x); + return (int)ret; +} + +static inline int __builtin_ctzll(unsigned long long x) { + unsigned long ret; + _BitScanForward64(&ret, x); + return (int)ret; +} + +static inline int __builtin_ctzl(unsigned long long x) { + return sizeof(x) == 8 ? __builtin_ctzll(x) : __builtin_ctz((uint32_t)x); } +#endif inline uptr getLeastSignificantSetBitIndex(uptr X) { DCHECK_NE(X, 0U); @@ -102,16 +142,20 @@ // Hardware specific inlinable functions. inline void yieldProcessor(u8 Count) { -#if defined(__i386__) || defined(__x86_64__) +#ifdef _MSC_VER + for (u8 I = 0; I < Count; I++) + YieldProcessor(); +#elif defined(__i386__) || defined(__x86_64__) __asm__ __volatile__("" ::: "memory"); for (u8 I = 0; I < Count; I++) __asm__ __volatile__("pause"); + __asm__ __volatile__("" ::: "memory"); #elif defined(__aarch64__) || defined(__arm__) __asm__ __volatile__("" ::: "memory"); for (u8 I = 0; I < Count; I++) __asm__ __volatile__("yield"); -#endif __asm__ __volatile__("" ::: "memory"); +#endif } // Platform specific functions. Index: compiler-rt/lib/scudo/standalone/crc32_hw.cpp =================================================================== --- compiler-rt/lib/scudo/standalone/crc32_hw.cpp +++ compiler-rt/lib/scudo/standalone/crc32_hw.cpp @@ -10,7 +10,12 @@ namespace scudo { -#if defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) +#ifdef _MSC_VER +#include + u32 computeHardwareCRC32(u32 Crc, uptr Data) { + return static_cast(_mm_crc32_u64(Crc, Data)); + } +#elif defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) u32 computeHardwareCRC32(u32 Crc, uptr Data) { return static_cast(CRC32_INTRINSIC(Crc, Data)); } Index: compiler-rt/lib/scudo/standalone/hostlib/CMakeLists.txt =================================================================== --- /dev/null +++ compiler-rt/lib/scudo/standalone/hostlib/CMakeLists.txt @@ -0,0 +1,71 @@ +# cmake for host library version of Scudo standalone build +project(scudo-sa) + +cmake_minimum_required(VERSION 3.13.4) + +set(CMAKE_GENERATOR_PLATFORM Visual Studio 15 2017) + +include_directories(.. ../include) + +set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS 1) + +add_library(scudo-sa STATIC + ../allocator_config.h + ../atomic_helpers.h + ../bytemap.h + ../checksum.h + ../chunk.h + ../combined.h + ../common.h + ../flags_parser.h + ../flags.h + ../fuchsia.h + ../internal_defs.h + ../linux.h + ../list.h + ../local_cache.h + ../memtag.h + ../mutex.h + ../options.h + ../platform.h + ../primary32.h + ../primary64.h + ../quarantine.h + ../release.h + ../report.h + ../secondary.h + ../size_class_map.h + ../stack_depot.h + ../stats.h + ../string_utils.h + ../tsd_exclusive.h + ../tsd_shared.h + ../tsd.h + ../vector.h + ../wind.h + ../windowsmmap.h + ../wrappers_c_checks.h + ../wrappers_c.h + + ../include/scudo/interface.h + + ../checksum.cpp + ../common.cpp + ../crc32_hw.cpp + ../flags_parser.cpp + ../flags.cpp + ../fuchsia.cpp + ../linux.cpp + ../release.cpp + ../report.cpp + ../string_utils.cpp + ../wind.cpp + ../windowsmmap.c + ../wrappers_c.cpp + ../wrappers_cpp.cpp + ) + +set_property(TARGET scudo-sa PROPERTY CXX_STANDARD 17) +set_property(TARGET scudo-sa PROPERTY MSVC_RUNTIME_LIBRARY "MultiThreaded") + +install(TARGETS scudo-sa DESTINATION ../../../../../SIE/lib) Index: compiler-rt/lib/scudo/standalone/hostlib/readme.txt =================================================================== --- /dev/null +++ compiler-rt/lib/scudo/standalone/hostlib/readme.txt @@ -0,0 +1,9 @@ +This directory holds the cmake file(s) for creation of a host library version +of scudo standalone. + +For Windows10 this can be generated by: + + cmake -G "Visual Studio 15 2017 Win64" -B build + +Then using Visual Studio to build the 'scudo-sa.sln' solution found in the +build directory. Index: compiler-rt/lib/scudo/standalone/include/scudo/interface.h =================================================================== --- compiler-rt/lib/scudo/standalone/include/scudo/interface.h +++ compiler-rt/lib/scudo/standalone/include/scudo/interface.h @@ -14,12 +14,12 @@ extern "C" { -__attribute__((weak)) const char *__scudo_default_options(); +WEAK const char *__scudo_default_options(); // Post-allocation & pre-deallocation hooks. // They must be thread-safe and not use heap related functions. -__attribute__((weak)) void __scudo_allocate_hook(void *ptr, size_t size); -__attribute__((weak)) void __scudo_deallocate_hook(void *ptr); +WEAK void __scudo_allocate_hook(void *ptr, size_t size); +WEAK void __scudo_deallocate_hook(void *ptr); void __scudo_print_stats(void); Index: compiler-rt/lib/scudo/standalone/internal_defs.h =================================================================== --- compiler-rt/lib/scudo/standalone/internal_defs.h +++ compiler-rt/lib/scudo/standalone/internal_defs.h @@ -19,6 +19,13 @@ #define ARRAY_SIZE(A) (sizeof(A) / sizeof((A)[0])) +// Single bit of correct size for shifting into pointers +#ifdef _MSC_VER +#define PTRBIT 1ULL +#else +#define PTRBIT 1UL +#endif + // String related macros. #define STRINGIFY_(S) #S @@ -28,6 +35,7 @@ // Attributes & builtins related macros. +#ifdef __GNUC__ #define INTERFACE __attribute__((visibility("default"))) #define HIDDEN __attribute__((visibility("hidden"))) #define WEAK __attribute__((weak)) @@ -66,7 +74,7 @@ // // If you're looking at this because your build failed, you probably introduced // a new member to scudo::Allocator<> (possibly transiently) that didn't have an -// initializer. The fix is easy - just add one. +// initializer. The fix is easy - just add one (GNU/clang) or remove (MSVC). #if defined(__has_attribute) #if __has_attribute(require_constant_initialization) #define SCUDO_REQUIRE_CONSTANT_INITIALIZATION \ @@ -75,6 +83,26 @@ #define SCUDO_REQUIRE_CONSTANT_INITIALIZATION #endif #endif +// for GNU and clang initialization ensures no dynamic initialisation +#define MAYBE_INIT(x) x +#else // __GNUC__ +#define WEAK +#define ALWAYS_INLINE __forceinline +#define NORETURN __declspec(noreturn) +#define NOINLINE __declspec(noinline) +#define FORMAT(F, A) +#ifndef UNUSED +#define UNUSED [[maybe_unused]] +#endif +#define NOUNIQUEADDRESS +#define INTERFACE +#define HIDDEN +#define PREFETCH(X) (X) +#define NOEXCEPT noexcept +#define SCUDO_REQUIRE_CONSTANT_INITIALIZATION +// for MSVC no initialization ensures no dynamic initialisation +#define MAYBE_INIT(x) +#endif // __GNUC__ namespace scudo { Index: compiler-rt/lib/scudo/standalone/local_cache.h =================================================================== --- compiler-rt/lib/scudo/standalone/local_cache.h +++ compiler-rt/lib/scudo/standalone/local_cache.h @@ -135,9 +135,9 @@ uptr ClassSize; CompactPtrT Chunks[2 * TransferBatch::MaxNumCached]; }; - PerClass PerClassArray[NumClasses] = {}; + PerClass PerClassArray[NumClasses] MAYBE_INIT(= {}); LocalStats Stats; - SizeClassAllocator *Allocator = nullptr; + SizeClassAllocator *Allocator MAYBE_INIT( = nullptr); ALWAYS_INLINE void initCacheMaybe(PerClass *C) { if (LIKELY(C->MaxCount)) Index: compiler-rt/lib/scudo/standalone/mutex.h =================================================================== --- compiler-rt/lib/scudo/standalone/mutex.h +++ compiler-rt/lib/scudo/standalone/mutex.h @@ -50,6 +50,8 @@ atomic_u32 M = {}; #elif SCUDO_FUCHSIA sync_mutex_t M = {}; +#elif SCUDO_WINDOWS + atomic_u32 M; #endif void lockSlow(); Index: compiler-rt/lib/scudo/standalone/platform.h =================================================================== --- compiler-rt/lib/scudo/standalone/platform.h +++ compiler-rt/lib/scudo/standalone/platform.h @@ -37,7 +37,13 @@ #define SCUDO_TRUSTY 0 #endif -#if __LP64__ +#if defined(_WIN32) +#define SCUDO_WINDOWS 1 +#else +#define SCUDO_WINDOWS 0 +#endif + +#if __LP64__ || defined(_MSC_VER) #define SCUDO_WORDSIZE 64U #else #define SCUDO_WORDSIZE 32U @@ -62,9 +68,9 @@ #endif #if defined(__aarch64__) -#define SCUDO_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 48) +#define SCUDO_MMAP_RANGE_SIZE FIRST_32_SECOND_64(PTRBIT << 32, PTRBIT << 48) #else -#define SCUDO_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47) +#define SCUDO_MMAP_RANGE_SIZE FIRST_32_SECOND_64(PTRBIT << 32, PTRBIT << 47) #endif // Older gcc have issues aligning to a constexpr, and require an integer. Index: compiler-rt/lib/scudo/standalone/primary32.h =================================================================== --- compiler-rt/lib/scudo/standalone/primary32.h +++ compiler-rt/lib/scudo/standalone/primary32.h @@ -46,7 +46,7 @@ // The bytemap can only track UINT8_MAX - 1 classes. static_assert(SizeClassMap::LargestClassId <= (UINT8_MAX - 1), ""); // Regions should be large enough to hold the largest Block. - static_assert((1UL << Config::PrimaryRegionSizeLog) >= SizeClassMap::MaxSize, + static_assert((PTRBIT << Config::PrimaryRegionSizeLog) >= SizeClassMap::MaxSize, ""); typedef SizeClassAllocator32 ThisT; typedef SizeClassAllocatorLocalCache CacheT; @@ -236,7 +236,7 @@ private: static const uptr NumClasses = SizeClassMap::NumClasses; - static const uptr RegionSize = 1UL << Config::PrimaryRegionSizeLog; + static const uptr RegionSize = PTRBIT << Config::PrimaryRegionSizeLog; static const uptr NumRegions = SCUDO_MMAP_RANGE_SIZE >> Config::PrimaryRegionSizeLog; static const u32 MaxNumBatches = SCUDO_ANDROID ? 4U : 8U; Index: compiler-rt/lib/scudo/standalone/primary64.h =================================================================== --- compiler-rt/lib/scudo/standalone/primary64.h +++ compiler-rt/lib/scudo/standalone/primary64.h @@ -225,7 +225,7 @@ const RegionInfo *RegionInfoArray = reinterpret_cast(RegionInfoData); uptr ClassId; - uptr MinDistance = -1UL; + uptr MinDistance = -(PTRBIT); for (uptr I = 0; I != NumClasses; ++I) { if (I == SizeClassMap::BatchClassId) continue; @@ -268,7 +268,7 @@ AtomicOptions Options; private: - static const uptr RegionSize = 1UL << Config::PrimaryRegionSizeLog; + static const uptr RegionSize = PTRBIT << Config::PrimaryRegionSizeLog; static const uptr NumClasses = SizeClassMap::NumClasses; static const uptr PrimarySize = RegionSize * NumClasses; @@ -291,14 +291,14 @@ struct UnpaddedRegionInfo { HybridMutex Mutex; SinglyLinkedList FreeList; - uptr RegionBeg = 0; - RegionStats Stats = {}; - u32 RandState = 0; - uptr MappedUser = 0; // Bytes mapped for user memory. - uptr AllocatedUser = 0; // Bytes allocated for user memory. - MapPlatformData Data = {}; - ReleaseToOsInfo ReleaseInfo = {}; - bool Exhausted = false; + uptr RegionBeg MAYBE_INIT(= 0); + RegionStats Stats MAYBE_INIT(= {}); + u32 RandState MAYBE_INIT(= 0); + uptr MappedUser MAYBE_INIT(= 0); // Bytes mapped for user memory. + uptr AllocatedUser MAYBE_INIT(= 0); // Bytes allocated for user memory. + MapPlatformData Data MAYBE_INIT(= {}); + ReleaseToOsInfo ReleaseInfo MAYBE_INIT(= {}); + bool Exhausted MAYBE_INIT(= false); }; struct RegionInfo : UnpaddedRegionInfo { char Padding[SCUDO_CACHE_LINE_SIZE - @@ -306,9 +306,9 @@ }; static_assert(sizeof(RegionInfo) % SCUDO_CACHE_LINE_SIZE == 0, ""); - uptr PrimaryBase = 0; - MapPlatformData Data = {}; - atomic_s32 ReleaseToOsIntervalMs = {}; + uptr PrimaryBase MAYBE_INIT(= 0); + MapPlatformData Data MAYBE_INIT(= {}); + atomic_s32 ReleaseToOsIntervalMs MAYBE_INIT(= {}); alignas(SCUDO_CACHE_LINE_SIZE) RegionInfo RegionInfoArray[NumClasses]; RegionInfo *getRegionInfo(uptr ClassId) { Index: compiler-rt/lib/scudo/standalone/quarantine.h =================================================================== --- compiler-rt/lib/scudo/standalone/quarantine.h +++ compiler-rt/lib/scudo/standalone/quarantine.h @@ -238,9 +238,9 @@ alignas(SCUDO_CACHE_LINE_SIZE) HybridMutex CacheMutex; CacheT Cache; alignas(SCUDO_CACHE_LINE_SIZE) HybridMutex RecycleMutex; - atomic_uptr MinSize = {}; - atomic_uptr MaxSize = {}; - alignas(SCUDO_CACHE_LINE_SIZE) atomic_uptr MaxCacheSize = {}; + atomic_uptr MinSize MAYBE_INIT(= {}); + atomic_uptr MaxSize MAYBE_INIT(= {}); + alignas(SCUDO_CACHE_LINE_SIZE) atomic_uptr MaxCacheSize MAYBE_INIT(= {}); void NOINLINE recycle(uptr MinSize, Callback Cb) { CacheT Tmp; Index: compiler-rt/lib/scudo/standalone/report.cpp =================================================================== --- compiler-rt/lib/scudo/standalone/report.cpp +++ compiler-rt/lib/scudo/standalone/report.cpp @@ -34,7 +34,13 @@ ScopedString Message; }; -inline void NORETURN trap() { __builtin_trap(); } +inline void NORETURN trap() { +#ifdef _MSC_VER + __debugbreak(); +#else + __builtin_trap(); +#endif +} // This could potentially be called recursively if a CHECK fails in the reports. void NORETURN reportCheckFailed(const char *File, int Line, Index: compiler-rt/lib/scudo/standalone/secondary.h =================================================================== --- compiler-rt/lib/scudo/standalone/secondary.h +++ compiler-rt/lib/scudo/standalone/secondary.h @@ -38,7 +38,7 @@ uptr CommitSize; uptr MapBase; uptr MapSize; - [[no_unique_address]] MapPlatformData Data; + NOUNIQUEADDRESS MapPlatformData Data; }; static_assert(sizeof(Header) % (1U << SCUDO_MIN_ALIGNMENT_LOG) == 0, ""); @@ -147,10 +147,10 @@ Entry.Data = H->Data; Entry.Time = Time; if (useMemoryTagging(Options)) { - if (Interval == 0 && !SCUDO_FUCHSIA) { + if (Interval == 0 && !SCUDO_FUCHSIA && !SCUDO_WINDOWS) { // Release the memory and make it inaccessible at the same time by // creating a new MAP_NOACCESS mapping on top of the existing mapping. - // Fuchsia does not support replacing mappings by creating a new mapping + // Fuchsia/windows does not support replacing mappings by creating a new mapping // on top so we just do the two syscalls there. Entry.Time = 0; mapSecondary(Options, Entry.CommitBase, Entry.CommitSize, @@ -160,12 +160,14 @@ &Entry.Data); } } else if (Interval == 0) { +#if !SCUDO_WINDOWS releasePagesToOS(Entry.CommitBase, 0, Entry.CommitSize, &Entry.Data); Entry.Time = 0; +#endif } do { ScopedLock L(Mutex); - if (useMemoryTagging(Options) && QuarantinePos == -1U) { + if (useMemoryTagging(Options) && QuarantinePos == static_cast(-1)) { // If we get here then memory tagging was disabled in between when we // read Options and when we locked Mutex. We can't insert our entry into // the quarantine or the cache because the permissions would be wrong so @@ -316,7 +318,7 @@ if (Entries[I].CommitBase) setMemoryPermission(Entries[I].CommitBase, Entries[I].CommitSize, 0, &Entries[I].Data); - QuarantinePos = -1U; + QuarantinePos = static_cast(-1); } void disable() { Mutex.lock(); } @@ -358,11 +360,12 @@ uptr MapBase; uptr MapSize; uptr BlockBegin; - [[no_unique_address]] MapPlatformData Data; + NOUNIQUEADDRESS MapPlatformData Data; u64 Time; }; void releaseIfOlderThan(CachedBlock &Entry, u64 Time) { +#if !SCUDO_WINDOWS if (!Entry.CommitBase || !Entry.Time) return; if (Entry.Time > Time) { @@ -372,9 +375,11 @@ } releasePagesToOS(Entry.CommitBase, 0, Entry.CommitSize, &Entry.Data); Entry.Time = 0; +#endif } void releaseOlderThan(u64 Time) { +#if !SCUDO_WINDOWS ScopedLock L(Mutex); if (!EntriesCount || OldestTime == 0 || OldestTime > Time) return; @@ -383,19 +388,20 @@ releaseIfOlderThan(Quarantine[I], Time); for (uptr I = 0; I < Config::SecondaryCacheEntriesArraySize; I++) releaseIfOlderThan(Entries[I], Time); +#endif } HybridMutex Mutex; - u32 EntriesCount = 0; - u32 QuarantinePos = 0; - atomic_u32 MaxEntriesCount = {}; - atomic_uptr MaxEntrySize = {}; - u64 OldestTime = 0; - u32 IsFullEvents = 0; - atomic_s32 ReleaseToOsIntervalMs = {}; - - CachedBlock Entries[Config::SecondaryCacheEntriesArraySize] = {}; - CachedBlock Quarantine[Config::SecondaryCacheQuarantineSize] = {}; + u32 EntriesCount MAYBE_INIT(= 0); + u32 QuarantinePos MAYBE_INIT(= 0); + atomic_u32 MaxEntriesCount MAYBE_INIT(= {}); + atomic_uptr MaxEntrySize MAYBE_INIT(= {}); + u64 OldestTime MAYBE_INIT(= 0); + u32 IsFullEvents MAYBE_INIT(= 0); + atomic_s32 ReleaseToOsIntervalMs MAYBE_INIT(= {}); + + CachedBlock Entries[Config::SecondaryCacheEntriesArraySize] MAYBE_INIT(= {}); + CachedBlock Quarantine[Config::SecondaryCacheQuarantineSize] MAYBE_INIT(= {}); }; template class MapAllocator { @@ -460,11 +466,11 @@ HybridMutex Mutex; DoublyLinkedList InUseBlocks; - uptr AllocatedBytes = 0; - uptr FreedBytes = 0; - uptr LargestSize = 0; - u32 NumberOfAllocs = 0; - u32 NumberOfFrees = 0; + uptr AllocatedBytes MAYBE_INIT(= 0); + uptr FreedBytes MAYBE_INIT(= 0); + uptr LargestSize MAYBE_INIT(= 0); + u32 NumberOfAllocs MAYBE_INIT(= 0); + u32 NumberOfFrees MAYBE_INIT(= 0); LocalStats Stats; }; @@ -484,8 +490,8 @@ uptr *BlockEndPtr, FillContentsMode FillContents) { if (Options.get(OptionBit::AddLargeAllocationSlack)) - Size += 1UL << SCUDO_MIN_ALIGNMENT_LOG; - Alignment = Max(Alignment, uptr(1U) << SCUDO_MIN_ALIGNMENT_LOG); + Size += PTRBIT << SCUDO_MIN_ALIGNMENT_LOG; + Alignment = Max(Alignment, PTRBIT << SCUDO_MIN_ALIGNMENT_LOG); const uptr PageSize = getPageSizeCached(); uptr RoundedSize = roundUpTo(roundUpTo(Size, Alignment) + LargeBlock::getHeaderSize() + @@ -602,11 +608,12 @@ template void MapAllocator::getStats(ScopedString *Str) const { - Str->append("Stats: MapAllocator: allocated %u times (%zuK), freed %u times " - "(%zuK), remains %u (%zuK) max %zuM\n", - NumberOfAllocs, AllocatedBytes >> 10, NumberOfFrees, - FreedBytes >> 10, NumberOfAllocs - NumberOfFrees, - (AllocatedBytes - FreedBytes) >> 10, LargestSize >> 20); + Str->append( + "Stats: MapAllocator: allocated %zu times (%zuK), freed %zu times " + "(%zuK), remains %zu (%zuK) max %zuM\n", + NumberOfAllocs, AllocatedBytes >> 10, NumberOfFrees, FreedBytes >> 10, + NumberOfAllocs - NumberOfFrees, (AllocatedBytes - FreedBytes) >> 10, + LargestSize >> 20); } } // namespace scudo Index: compiler-rt/lib/scudo/standalone/size_class_map.h =================================================================== --- compiler-rt/lib/scudo/standalone/size_class_map.h +++ compiler-rt/lib/scudo/standalone/size_class_map.h @@ -17,7 +17,7 @@ inline uptr scaledLog2(uptr Size, uptr ZeroLog, uptr LogBits) { const uptr L = getMostSignificantSetBitIndex(Size); - const uptr LBits = (Size >> (L - LogBits)) - (1 << LogBits); + const uptr LBits = (Size >> (L - LogBits)) - (PTRBIT << LogBits); const uptr HBits = (L - ZeroLog) << LogBits; return LBits + HBits; } @@ -58,16 +58,16 @@ class FixedSizeClassMap : public SizeClassMapBase { typedef SizeClassMapBase Base; - static const uptr MinSize = 1UL << Config::MinSizeLog; - static const uptr MidSize = 1UL << Config::MidSizeLog; + static const uptr MinSize = PTRBIT << Config::MinSizeLog; + static const uptr MidSize = PTRBIT << Config::MidSizeLog; static const uptr MidClass = MidSize / MinSize; static const u8 S = Config::NumBits - 1; - static const uptr M = (1UL << S) - 1; + static const uptr M = (PTRBIT << S) - 1; public: static const u32 MaxNumCachedHint = Config::MaxNumCachedHint; - static const uptr MaxSize = (1UL << Config::MaxSizeLog) + Config::SizeDelta; + static const uptr MaxSize = (PTRBIT << Config::MaxSizeLog) + Config::SizeDelta; static const uptr NumClasses = MidClass + ((Config::MaxSizeLog - Config::MidSizeLog) << S) + 1; static_assert(NumClasses <= 256, ""); @@ -110,7 +110,7 @@ typedef SizeClassMapBase Base; static const u8 S = Config::NumBits - 1; - static const uptr M = (1UL << S) - 1; + static const uptr M = (PTRBIT << S) - 1; static const uptr ClassesSize = sizeof(Config::Classes) / sizeof(Config::Classes[0]); Index: compiler-rt/lib/scudo/standalone/tsd.h =================================================================== --- compiler-rt/lib/scudo/standalone/tsd.h +++ compiler-rt/lib/scudo/standalone/tsd.h @@ -14,7 +14,9 @@ #include "mutex.h" #include // for PTHREAD_DESTRUCTOR_ITERATIONS +#ifndef _WIN32 #include +#endif // With some build setups, this might still not be defined. #ifndef PTHREAD_DESTRUCTOR_ITERATIONS Index: compiler-rt/lib/scudo/standalone/tsd_shared.h =================================================================== --- compiler-rt/lib/scudo/standalone/tsd_shared.h +++ compiler-rt/lib/scudo/standalone/tsd_shared.h @@ -116,7 +116,7 @@ } ALWAYS_INLINE TSD *getCurrentTSD() { - return reinterpret_cast *>(*getTlsPtr() & ~1ULL); + return reinterpret_cast *>(*getTlsPtr() & ~(PTRBIT)); } bool setNumberOfTSDs(u32 N) { @@ -146,8 +146,8 @@ } void setDisableMemInit(bool B) { - *getTlsPtr() &= ~1ULL; - *getTlsPtr() |= B; + *getTlsPtr() &= ~(PTRBIT); + *getTlsPtr() |= static_cast(B); } NOINLINE void initThread(Allocator *Instance) { @@ -201,11 +201,11 @@ return CurrentTSD; } - atomic_u32 CurrentIndex = {}; - u32 NumberOfTSDs = 0; - u32 NumberOfCoPrimes = 0; - u32 CoPrimes[TSDsArraySize] = {}; - bool Initialized = false; + atomic_u32 CurrentIndex MAYBE_INIT(= {}); + u32 NumberOfTSDs MAYBE_INIT(= 0); + u32 NumberOfCoPrimes MAYBE_INIT(= 0); + u32 CoPrimes[TSDsArraySize] MAYBE_INIT(= {}); + bool Initialized MAYBE_INIT(= false); HybridMutex Mutex; HybridMutex MutexTSDs; TSD TSDs[TSDsArraySize]; Index: compiler-rt/lib/scudo/standalone/wind.h =================================================================== --- /dev/null +++ compiler-rt/lib/scudo/standalone/wind.h @@ -0,0 +1,43 @@ +//===-- wind.h -------------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_WIND_H_ +#define SCUDO_WIND_H_ + +#include "platform.h" + +#include + +#if SCUDO_WINDOWS + +typedef INIT_ONCE pthread_once_t; +#define PTHREAD_ONCE_INIT 0 +#define __ATOMIC_SEQ_CST memory_order_seq_cst + +namespace scudo { + +// MapPlatformData is unused on Windows, define it as a minimally sized structure. +struct MapPlatformData {}; + +// pthread is not supplied on windows +struct pthread_key_t +{ + DWORD key; + void (*destructor) (void *); + void* val; +}; +int pthread_once(pthread_once_t * once_control, void(*init_routine) (void)); +int pthread_key_create(pthread_key_t *key, void(*destructor)(void*)); +void *pthread_getspecific(pthread_key_t key); +int pthread_setspecific(pthread_key_t* key, void *value); + +} // namespace scudo + +#endif // SCUDO_WINDOWS + +#endif // SCUDO_WIND_H_ Index: compiler-rt/lib/scudo/standalone/wind.cpp =================================================================== --- /dev/null +++ compiler-rt/lib/scudo/standalone/wind.cpp @@ -0,0 +1,198 @@ +//===-- wind.cpp -----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "platform.h" + +#if SCUDO_WINDOWS + +#include "atomic_helpers.h" +#include "common.h" +#include "mutex.h" +#include "wind.h" + +extern "C" { +#include "WindowsMMap.h" +}; + +#include +#include +#include +#include + +namespace scudo { + + uptr getPageSize() { + SYSTEM_INFO si; + GetSystemInfo(&si); + return static_cast(si.dwAllocationGranularity); + } + + const char *getEnv(const char *Name) { + char *pValue; + size_t len; + errno_t err = _dupenv_s(&pValue, &len, Name); + if (err) + return NULL; + return pValue; // we leak here? + // return getenv(Name); + } + + u64 getMonotonicTime() { + return GetTickCount64() * 1000000ULL; + } + + u32 getThreadID() { + return GetCurrentThreadId(); + } + + u32 getNumberOfCPUs() { + SYSTEM_INFO si; + GetSystemInfo(&si); + return static_cast(si.dwNumberOfProcessors); + } + + bool getRandom(void *Buffer, uptr Length, UNUSED bool Blocking) { + if (!Buffer || !Length || Length > MaxRandomLength) + return false; + unsigned int* toWrite = static_cast(Buffer); + while (Length > sizeof(unsigned int)) { + *toWrite = rand(); + toWrite++; + Length = Length - sizeof(unsigned int); + } + return true; + } + + int pthread_once(pthread_once_t * once_control, void (*init_routine) (void)) + { + return InitOnceExecuteOnce(once_control, (PINIT_ONCE_FN)init_routine, NULL, NULL); + } + + int pthread_key_create(pthread_key_t *key, void(*destructor)(void*)) + { + key->destructor = destructor; + key->val = NULL; + return 0; + } + void* pthread_getspecific(pthread_key_t key) + { + return key.val; + } + int pthread_setspecific(pthread_key_t* key, void *value) + { + key->val = static_cast(value); + return 0; + } + + + void outputRaw(const char *Buffer) { + (void)std::fputs(Buffer, stderr); + }; + + void die() { abort(); } + + void *map(void *Addr, uptr Size, UNUSED const char *Name, uptr Flags, + UNUSED MapPlatformData *Data) { + int MmapFlags = MAP_PRIVATE | MAP_ANONYMOUS; + int MmapProt; + if (Flags & MAP_NOACCESS) { + MmapProt = PROT_NONE; + } + else { + MmapProt = PROT_READ | PROT_WRITE; + } +#if 0 + if (Addr) { + // Currently no scenario for a noaccess mapping with a fixed address. + DCHECK_EQ(Flags & MAP_NOACCESS, 0); + MmapFlags |= MAP_FIXED; + } +#endif + void *P = mmap(Addr, Size, MmapProt, MmapFlags, -1, 0); + if (P == MAP_FAILED) { + if (!(Flags & MAP_ALLOWNOMEM) || errno != ENOMEM) + dieOnMapUnmapError(errno == ENOMEM); + return nullptr; + } + return P; + } + + void unmap(void *Addr, uptr Size, UNUSED uptr Flags, + UNUSED MapPlatformData *Data) { + if (munmap(Addr, static_cast(Size)) != 0) + dieOnMapUnmapError(); + } + + void setMemoryPermission(uptr Addr, uptr Size, uptr Flags, + UNUSED MapPlatformData *Data) { + assert(FALSE && "Cannot alter memory permission on windows"); +#if 0 + int Prot = (Flags & MAP_NOACCESS) ? PROT_NONE : (PROT_READ | PROT_WRITE); + if (mprotect(reinterpret_cast(Addr), Size, Prot) != 0) + dieOnMapUnmapError(); +#endif + } + + void releasePagesToOS(uptr BaseAddress, uptr Offset, uptr Size, + UNUSED MapPlatformData *Data) { + // windows seems to have no direct method to imitate the linux + // madvise feature. One side-effect of madvise is that the + // block is zero-filled. scudo relies on this when reusing a + // 'released' block. So just emulate that here. + memset(reinterpret_cast(BaseAddress+Offset), 0, Size); + }; + + namespace { + enum State : u32 { Unlocked = 0, Locked = 1, Sleeping = 2 }; + } + + bool HybridMutex::tryLock() { + return atomic_compare_exchange(&M, Unlocked, Locked) == Unlocked; + } + + // The following is based on https://akkadia.org/drepper/futex.pdf. + void HybridMutex::lockSlow() { + u32 V = atomic_compare_exchange(&M, Unlocked, Locked); + if (V == Unlocked) + return; + if (V != Sleeping) + V = atomic_exchange(&M, Sleeping, memory_order_acquire); + while (V != Unlocked) { +// syscall(SYS_futex, reinterpret_cast(&M), FUTEX_WAIT_PRIVATE, Sleeping, +// nullptr, nullptr, 0); + u32 compare = Sleeping; + WaitOnAddress(&M, &compare, sizeof(compare), INFINITE); + V = atomic_exchange(&M, Sleeping, memory_order_acquire); + } + } + + void HybridMutex::unlock() { + if (atomic_fetch_sub(&M, 1U, memory_order_release) != Locked) { + atomic_store(&M, Unlocked, memory_order_release); + WakeByAddressSingle(&M); +// syscall(SYS_futex, reinterpret_cast(&M), FUTEX_WAKE_PRIVATE, 1, +// nullptr, nullptr, 0); + } + } + + void setAbortMessage(const char *Message) { + } + +} // namespace scudo + +// MSVC does not expose a 'manual' weak function scheme. It defaults to allowing +// overriding of a function in a library with no extra notation. These are the +// overridable weak hooks. + +extern "C" void __scudo_allocate_hook(void *, size_t) {}; +extern "C" void __scudo_deallocate_hook(void *ptr) {}; +extern "C" const char *__scudo_default_options() { return ""; }; + +#pragma comment(lib, "Synchronization.lib") + +#endif Index: compiler-rt/lib/scudo/standalone/windowsmmap.h =================================================================== --- /dev/null +++ compiler-rt/lib/scudo/standalone/windowsmmap.h @@ -0,0 +1,68 @@ +/*===- WindowsMMap.h - Support library for PGO instrumentation ------------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#ifndef PROFILE_INSTRPROFILING_WINDOWS_MMAP_H +#define PROFILE_INSTRPROFILING_WINDOWS_MMAP_H + +#if defined(_WIN32) + +#include +#include +#include + +/* + * mmap() flags + */ +#define PROT_READ 0x1 +#define PROT_WRITE 0x2 +#define PROT_EXEC 0x4 +#define PROT_NONE 0x0 + +#define MAP_FILE 0x00 +#define MAP_SHARED 0x01 +#define MAP_PRIVATE 0x02 +#define MAP_ANONYMOUS 0x20 +#define MAP_ANON MAP_ANONYMOUS +#define MAP_FAILED ((void *) -1) + +/* + * msync() flags + */ +#define MS_ASYNC 0x0001 /* return immediately */ +#define MS_INVALIDATE 0x0002 /* invalidate all cached data */ +#define MS_SYNC 0x0010 /* msync synchronously */ + +/* + * flock() operations + */ +#define LOCK_SH 1 /* shared lock */ +#define LOCK_EX 2 /* exclusive lock */ +#define LOCK_NB 4 /* don't block when locking */ +#define LOCK_UN 8 /* unlock */ + +#ifdef __USE_FILE_OFFSET64 +# define DWORD_HI(x) (x >> 32) +# define DWORD_LO(x) ((x) & 0xffffffff) +#else +# define DWORD_HI(x) (0) +# define DWORD_LO(x) (x) +#endif + + void *mmap(void *start, size_t length, int prot, int flags, int fd, + off_t offset); + + int munmap(void *addr, size_t length); + + int msync(void *addr, size_t length, int flags); + + int flock(int fd, int operation); + + +#endif /* _WIN32 */ + +#endif /* PROFILE_INSTRPROFILING_WINDOWS_MMAP_H */ Index: compiler-rt/lib/scudo/standalone/windowsmmap.c =================================================================== --- /dev/null +++ compiler-rt/lib/scudo/standalone/windowsmmap.c @@ -0,0 +1,198 @@ +/* + * This code is derived from uClibc (original license follows). + * https://git.uclibc.org/uClibc/tree/utils/mmap-windows.c + */ + /* mmap() replacement for Windows + * + * Author: Mike Frysinger + * Placed into the public domain + */ + +/* References: + * CreateFileMapping: http://msdn.microsoft.com/en-us/library/aa366537(VS.85).aspx + * CloseHandle: http://msdn.microsoft.com/en-us/library/ms724211(VS.85).aspx + * MapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366761(VS.85).aspx + * UnmapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366882(VS.85).aspx + */ + +#if defined(_WIN32) + +#include "WindowsMMap.h" + +#define WIN32_LEAN_AND_MEAN +#include + +//#include "InstrProfiling.h" +#define COMPILER_RT_VISIBILITY + +COMPILER_RT_VISIBILITY +void *mmap(void *start, size_t length, int prot, int flags, int fd, off_t offset) +{ + if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC)) + return MAP_FAILED; + if (fd == -1) { + if (!(flags & MAP_ANON) || offset) + return MAP_FAILED; + } else if (flags & MAP_ANON) + return MAP_FAILED; + + DWORD flAlloc = MEM_COMMIT; + DWORD flProtect; + if (prot & PROT_WRITE) { + if (prot & PROT_EXEC) + flProtect = PAGE_EXECUTE_READWRITE; + else + flProtect = PAGE_READWRITE; + } + else if (prot & PROT_EXEC) { + if (prot & PROT_READ) + flProtect = PAGE_EXECUTE_READ; + else if (prot & PROT_EXEC) + flProtect = PAGE_EXECUTE; + } + else if (prot & PROT_READ) + flProtect = PAGE_READONLY; + else + { + flProtect = PAGE_READONLY | PAGE_GUARD; + flAlloc = MEM_RESERVE; + } + + size_t end = length + offset; + + void* ret = VirtualAlloc(start, end, flAlloc, flProtect); + + if (ret == NULL) { + int err = GetLastError(); + return MAP_FAILED; + } + return ret; +#if 0 + HANDLE mmap_fd, h; + if (fd == -1) + mmap_fd = INVALID_HANDLE_VALUE; + else + mmap_fd = (HANDLE)_get_osfhandle(fd); + h = CreateFileMapping(mmap_fd, NULL, flProtect, DWORD_HI(end), DWORD_LO(end), NULL); + if (h == NULL) { + int err = GetLastError(); + return MAP_FAILED; + } + + DWORD dwDesiredAccess; + if (prot & PROT_WRITE) + dwDesiredAccess = FILE_MAP_WRITE; + else + dwDesiredAccess = FILE_MAP_READ; + if (prot & PROT_EXEC) + dwDesiredAccess |= FILE_MAP_EXECUTE; + if (flags & MAP_PRIVATE) + dwDesiredAccess |= FILE_MAP_COPY; + void *ret = MapViewOfFile(h, dwDesiredAccess, DWORD_HI(offset), DWORD_LO(offset), length); + if (ret == NULL) { + CloseHandle(h); + ret = MAP_FAILED; + } + return ret; +#endif +} + +COMPILER_RT_VISIBILITY +int munmap(void *addr, size_t length) +{ + return VirtualFree(addr, length, MEM_RELEASE); + //return UnmapViewOfFile(addr); + /* ruh-ro, we leaked handle from CreateFileMapping() ... */ +} + +COMPILER_RT_VISIBILITY +int msync(void *addr, size_t length, int flags) +{ + if (flags & MS_INVALIDATE) + return -1; /* Not supported. */ + + /* Exactly one of MS_ASYNC or MS_SYNC must be specified. */ + switch (flags & (MS_ASYNC | MS_SYNC)) { + case MS_SYNC: + case MS_ASYNC: + break; + default: + return -1; + } + + if (!FlushViewOfFile(addr, length)) + return -1; + + if (flags & MS_SYNC) { + /* FIXME: No longer have access to handle from CreateFileMapping(). */ + /* + * if (!FlushFileBuffers(h)) + * return -1; + */ + } + + return 0; +} + +COMPILER_RT_VISIBILITY +int lock(HANDLE handle, DWORD lockType, BOOL blocking) { + DWORD flags = lockType; + if (!blocking) + flags |= LOCKFILE_FAIL_IMMEDIATELY; + + OVERLAPPED overlapped; + ZeroMemory(&overlapped, sizeof(OVERLAPPED)); + overlapped.hEvent = CreateEvent(NULL, FALSE, FALSE, NULL); + BOOL result = LockFileEx(handle, flags, 0, MAXDWORD, MAXDWORD, &overlapped); + if (!result) { + DWORD dw = GetLastError(); + + // In non-blocking mode, return an error if the file is locked. + if (!blocking && dw == ERROR_LOCK_VIOLATION) + return -1; // EWOULDBLOCK + + // If the error is ERROR_IO_PENDING, we need to wait until the operation + // finishes. Otherwise, we return an error. + if (dw != ERROR_IO_PENDING) + return -1; + + DWORD dwNumBytes; + if (!GetOverlappedResult(handle, &overlapped, &dwNumBytes, TRUE)) + return -1; + } + + return 0; +} + +COMPILER_RT_VISIBILITY +int flock(int fd, int operation) { + HANDLE handle = (HANDLE)_get_osfhandle(fd); + if (handle == INVALID_HANDLE_VALUE) + return -1; + + BOOL blocking = (operation & LOCK_NB) == 0; + int op = operation & ~LOCK_NB; + + switch (op) { + case LOCK_EX: + return lock(handle, LOCKFILE_EXCLUSIVE_LOCK, blocking); + + case LOCK_SH: + return lock(handle, 0, blocking); + + case LOCK_UN: + if (!UnlockFile(handle, 0, 0, MAXDWORD, MAXDWORD)) + return -1; + break; + + default: + return -1; + } + + return 0; +} + +#undef DWORD_HI +#undef DWORD_LO + +#endif /* _WIN32 */ Index: compiler-rt/lib/scudo/standalone/wrappers_c.cpp =================================================================== --- compiler-rt/lib/scudo/standalone/wrappers_c.cpp +++ compiler-rt/lib/scudo/standalone/wrappers_c.cpp @@ -18,6 +18,12 @@ #include #include +#ifdef _WIN32 +// win32 headers use this macro +#undef INTERFACE +#define INTERFACE +#endif + #define SCUDO_PREFIX(name) name #define SCUDO_ALLOCATOR Allocator Index: compiler-rt/lib/scudo/standalone/wrappers_c.inc =================================================================== --- compiler-rt/lib/scudo/standalone/wrappers_c.inc +++ compiler-rt/lib/scudo/standalone/wrappers_c.inc @@ -151,8 +151,10 @@ void SCUDO_PREFIX(malloc_postinit)() { SCUDO_ALLOCATOR.initGwpAsan(); +#ifndef _WIN32 pthread_atfork(SCUDO_PREFIX(malloc_disable), SCUDO_PREFIX(malloc_enable), SCUDO_PREFIX(malloc_enable)); +#endif } INTERFACE WEAK int SCUDO_PREFIX(mallopt)(int param, int value) { @@ -212,6 +214,7 @@ } INTERFACE WEAK int SCUDO_PREFIX(malloc_info)(UNUSED int options, FILE *stream) { +#ifndef _MSC_VER // FIXME: const scudo::uptr max_size = decltype(SCUDO_ALLOCATOR)::PrimaryT::SizeClassMap::MaxSize; auto *sizes = static_cast( @@ -229,6 +232,7 @@ fprintf(stream, "\n", i, sizes[i]); fputs("\n", stream); SCUDO_PREFIX(free)(sizes); +#endif return 0; } Index: compiler-rt/lib/scudo/standalone/wrappers_cpp.cpp =================================================================== --- compiler-rt/lib/scudo/standalone/wrappers_cpp.cpp +++ compiler-rt/lib/scudo/standalone/wrappers_cpp.cpp @@ -15,13 +15,21 @@ #include +#ifdef _WIN32 +// win32 headers use this macro +#undef INTERFACE +#define INTERFACE +#endif + extern "C" void malloc_postinit(); extern HIDDEN scudo::Allocator Allocator; +#ifndef _WIN32 namespace std { struct nothrow_t {}; enum class align_val_t : size_t {}; } // namespace std +#endif INTERFACE WEAK void *operator new(size_t size) { return Allocator.allocate(size, scudo::Chunk::Origin::New);