Index: compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator_primary64.h =================================================================== --- compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator_primary64.h +++ compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator_primary64.h @@ -62,10 +62,10 @@ // as a 4-byte integer (offset from the region start shifted right by 4). typedef u32 CompactPtrT; static const uptr kCompactPtrScale = 4; - CompactPtrT PointerToCompactPtr(uptr base, uptr ptr) { + CompactPtrT PointerToCompactPtr(uptr base, uptr ptr) const { return static_cast((ptr - base) >> kCompactPtrScale); } - uptr CompactPtrToPointer(uptr base, CompactPtrT ptr32) { + uptr CompactPtrToPointer(uptr base, CompactPtrT ptr32) const { return base + (static_cast(ptr32) << kCompactPtrScale); } @@ -155,7 +155,7 @@ space_beg; } - uptr GetRegionBeginBySizeClass(uptr class_id) { + uptr GetRegionBeginBySizeClass(uptr class_id) const { return SpaceBeg() + kRegionSize * class_id; } @@ -294,7 +294,240 @@ static const uptr kNumClasses = SizeClassMap::kNumClasses; static const uptr kNumClassesRounded = SizeClassMap::kNumClassesRounded; + // A packed array of counters. Each counter occupies 2^n bits, enough to store + // counter's max_value. Ctor will try to allocate the required buffer via + // mapper->MapPackedCounterArrayBuffer and the caller is expected to check + // whether the initialization was successful by checking IsAllocated() result. + // For the performance sake, none of the accessors check the validity of the + // arguments, it is assumed that index is always in [0, n) range and the value + // is not incremented past max_value. + template + class PackedCounterArray { + public: + PackedCounterArray(u64 num_counters, u64 max_value, MemoryMapperT *mapper) + : n(num_counters), memory_mapper(mapper) { + CHECK_GT(num_counters, 0); + CHECK_GT(max_value, 0); + constexpr u64 kMaxCounterBits = sizeof(*buffer) * 8ULL; + // Rounding counter storage size up to the power of two allows for using + // bit shifts calculating particular counter's index and offset. + uptr counter_size_bits = + RoundUpToPowerOfTwo(MostSignificantSetBitIndex(max_value) + 1); + CHECK_LE(counter_size_bits, kMaxCounterBits); + counter_size_bits_log = Log2(counter_size_bits); + counter_mask = ~0ULL >> (kMaxCounterBits - counter_size_bits); + + uptr packing_ratio = kMaxCounterBits >> counter_size_bits_log; + CHECK_GT(packing_ratio, 0); + packing_ratio_log = Log2(packing_ratio); + bit_offset_mask = packing_ratio - 1; + + buffer_size = + (RoundUpTo(n, 1ULL << packing_ratio_log) >> packing_ratio_log) * + sizeof(*buffer); + buffer = reinterpret_cast( + memory_mapper->MapPackedCounterArrayBuffer(buffer_size)); + } + ~PackedCounterArray() { + if (buffer) { + memory_mapper->UnmapPackedCounterArrayBuffer( + reinterpret_cast(buffer), buffer_size); + } + } + + bool IsAllocated() const { + return !!buffer; + } + + u64 GetCount() const { + return n; + } + + uptr Get(uptr i) const { + DCHECK_LT(i, n); + uptr index = i >> packing_ratio_log; + uptr bit_offset = (i & bit_offset_mask) << counter_size_bits_log; + return (buffer[index] >> bit_offset) & counter_mask; + } + + void Inc(uptr i) const { + DCHECK_LT(Get(i), counter_mask); + uptr index = i >> packing_ratio_log; + uptr bit_offset = (i & bit_offset_mask) << counter_size_bits_log; + buffer[index] += 1ULL << bit_offset; + } + + void IncRange(uptr from, uptr to) const { + DCHECK_LE(from, to); + for (uptr i = from; i <= to; i++) + Inc(i); + } + + private: + const u64 n; + u64 counter_size_bits_log; + u64 counter_mask; + u64 packing_ratio_log; + u64 bit_offset_mask; + + MemoryMapperT* const memory_mapper; + u64 buffer_size; + u64* buffer; + }; + + template + class FreePagesRangeTracker { + public: + explicit FreePagesRangeTracker(MemoryMapperT* mapper) + : memory_mapper(mapper), + page_size_scaled_log(Log2(GetPageSizeCached() >> kCompactPtrScale)), + in_the_range(false), current_page(0), current_range_start_page(0) {} + + void NextPage(bool freed) { + if (freed) { + if (!in_the_range) { + current_range_start_page = current_page; + in_the_range = true; + } + } else { + CloseOpenedRange(); + } + current_page++; + } + + void Done() { + CloseOpenedRange(); + } + + private: + void CloseOpenedRange() { + if (in_the_range) { + memory_mapper->ReleasePageRangeToOS( + current_range_start_page << page_size_scaled_log, + current_page << page_size_scaled_log); + in_the_range = false; + } + } + + MemoryMapperT* const memory_mapper; + const uptr page_size_scaled_log; + bool in_the_range; + uptr current_page; + uptr current_range_start_page; + }; + + // Iterates over the free_array to identify memory pages containing freed + // chunks only and returns these pages back to OS. + // allocated_pages_count is the total number of pages allocated for the + // current bucket. + template + static void ReleaseFreeMemoryToOS(CompactPtrT *free_array, + uptr free_array_count, uptr chunk_size, + uptr allocated_pages_count, + MemoryMapperT *memory_mapper) { + const uptr page_size = GetPageSizeCached(); + + // Figure out the number of chunks per page and whether we can take a fast + // path (the number of chunks per page is the same for all pages). + uptr full_pages_chunk_count_max; + bool same_chunk_count_per_page; + if (chunk_size <= page_size && page_size % chunk_size == 0) { + // Same number of chunks per page, no cross overs. + full_pages_chunk_count_max = page_size / chunk_size; + same_chunk_count_per_page = true; + } else if (chunk_size <= page_size && page_size % chunk_size != 0 && + chunk_size % (page_size % chunk_size) == 0) { + // Some chunks are crossing page boundaries, which means that the page + // contains one or two partial chunks, but all pages contain the same + // number of chunks. + full_pages_chunk_count_max = page_size / chunk_size + 1; + same_chunk_count_per_page = true; + } else if (chunk_size <= page_size) { + // Some chunks are crossing page boundaries, which means that the page + // contains one or two partial chunks. + full_pages_chunk_count_max = page_size / chunk_size + 2; + same_chunk_count_per_page = false; + } else if (chunk_size > page_size && chunk_size % page_size == 0) { + // One chunk covers multiple pages, no cross overs. + full_pages_chunk_count_max = 1; + same_chunk_count_per_page = true; + } else if (chunk_size > page_size) { + // One chunk covers multiple pages, Some chunks are crossing page + // boundaries. Some pages contain one chunk, some contain two. + full_pages_chunk_count_max = 2; + same_chunk_count_per_page = false; + } else { + UNREACHABLE("All chunk_size/page_size ratios must be handled."); + } + + PackedCounterArray counters(allocated_pages_count, + full_pages_chunk_count_max, + memory_mapper); + if (!counters.IsAllocated()) + return; + + const uptr chunk_size_scaled = chunk_size >> kCompactPtrScale; + const uptr page_size_scaled = page_size >> kCompactPtrScale; + const uptr page_size_scaled_log = Log2(page_size_scaled); + + // Iterate over free chunks and count how many free chunks affect each + // allocated page. + if (chunk_size <= page_size && page_size % chunk_size == 0) { + // Each chunk affects one page only. + for (uptr i = 0; i < free_array_count; i++) + counters.Inc(free_array[i] >> page_size_scaled_log); + } else { + // In all other cases chunks might affect more than one page. + for (uptr i = 0; i < free_array_count; i++) { + counters.IncRange( + free_array[i] >> page_size_scaled_log, + (free_array[i] + chunk_size_scaled - 1) >> page_size_scaled_log); + } + } + + // Iterate over pages detecting ranges of pages with chunk counters equal + // to the expected number of chunks for the particular page. + FreePagesRangeTracker range_tracker(memory_mapper); + if (same_chunk_count_per_page) { + // Fast path, every page has the same number of chunks affecting it. + for (uptr i = 0; i < counters.GetCount(); i++) + range_tracker.NextPage(counters.Get(i) == full_pages_chunk_count_max); + } else { + // Show path, go through the pages keeping count how many chunks affect + // each page. + const uptr pn = + chunk_size < page_size ? page_size_scaled / chunk_size_scaled : 1; + const uptr pnc = pn * chunk_size_scaled; + // The idea is to increment the current page pointer by the first chunk + // size, middle portion size (the portion of the page covered by chunks + // except the first and the last one) and then the last chunk size, adding + // up the number of chunks on the current page and checking on every step + // whether the page boundary was crossed. + uptr prev_page_boundary = 0; + uptr current_boundary = 0; + for (uptr i = 0; i < counters.GetCount(); i++) { + uptr page_boundary = prev_page_boundary + page_size_scaled; + uptr chunks_per_page = pn; + if (current_boundary < page_boundary) { + if (current_boundary > prev_page_boundary) + chunks_per_page++; + current_boundary += pnc; + if (current_boundary < page_boundary) { + chunks_per_page++; + current_boundary += chunk_size_scaled; + } + } + prev_page_boundary = page_boundary; + + range_tracker.NextPage(counters.Get(i) == chunks_per_page); + } + } + range_tracker.Done(); + } + private: + friend class MemoryMapper; + static const uptr kRegionSize = kSpaceSize / kNumClassesRounded; // FreeArray is the array of free-d chunks (stored as 4-byte offsets). // In the worst case it may reguire kRegionSize/SizeClassMap::kMinSize @@ -359,18 +592,18 @@ Swap(a[i], a[RandN(rand_state, i + 1)]); } - RegionInfo *GetRegionInfo(uptr class_id) { + RegionInfo *GetRegionInfo(uptr class_id) const { CHECK_LT(class_id, kNumClasses); RegionInfo *regions = reinterpret_cast(SpaceBeg() + kSpaceSize); return ®ions[class_id]; } - uptr GetMetadataEnd(uptr region_beg) { + uptr GetMetadataEnd(uptr region_beg) const { return region_beg + kRegionSize - kFreeArraySize; } - uptr GetChunkIdx(uptr chunk, uptr size) { + uptr GetChunkIdx(uptr chunk, uptr size) const { if (!kUsingConstantSpaceBeg) chunk -= SpaceBeg(); @@ -382,9 +615,8 @@ return (u32)offset / (u32)size; } - CompactPtrT *GetFreeArray(uptr region_beg) { - return reinterpret_cast(region_beg + kRegionSize - - kFreeArraySize); + CompactPtrT *GetFreeArray(uptr region_beg) const { + return reinterpret_cast(GetMetadataEnd(region_beg)); } bool MapWithCallback(uptr beg, uptr size) { @@ -410,8 +642,8 @@ uptr num_freed_chunks) { uptr needed_space = num_freed_chunks * sizeof(CompactPtrT); if (region->mapped_free_array < needed_space) { - CHECK_LE(needed_space, kFreeArraySize); uptr new_mapped_free_array = RoundUpTo(needed_space, kFreeArrayMapSize); + CHECK_LE(new_mapped_free_array, kFreeArraySize); uptr current_map_end = reinterpret_cast(GetFreeArray(region_beg)) + region->mapped_free_array; uptr new_map_size = new_mapped_free_array - region->mapped_free_array; @@ -495,22 +727,54 @@ CHECK_LE(region->allocated_meta, region->mapped_meta); region->exhausted = false; + // TODO(alekseyshl): Consider bumping last_release_at_ns here to prevent + // MaybeReleaseToOS from releasing just allocated pages or protect these + // not yet used chunks some other way. + return true; } - void MaybeReleaseChunkRange(uptr region_beg, uptr chunk_size, - CompactPtrT first, CompactPtrT last) { - uptr beg_ptr = CompactPtrToPointer(region_beg, first); - uptr end_ptr = CompactPtrToPointer(region_beg, last) + chunk_size; - ReleaseMemoryPagesToOS(beg_ptr, end_ptr); - } + class MemoryMapper { + public: + MemoryMapper(const ThisT& base_allocator, uptr class_id) + : allocator(base_allocator), + region_base(base_allocator.GetRegionBeginBySizeClass(class_id)), + released_ranges_count(0) { + } + + uptr GetReleasedRangesCount() const { + return released_ranges_count; + } + + uptr MapPackedCounterArrayBuffer(uptr buffer_size) { + // TODO(alekseyshl): The idea to explore is to check if we have enough + // space between num_freed_chunks*sizeof(CompactPtrT) and + // mapped_free_array to fit buffer_size bytes and use that space instead + // of mapping a temporary one. + return reinterpret_cast( + MmapOrDieOnFatalError(buffer_size, "ReleaseToOSPageCounters")); + } + + void UnmapPackedCounterArrayBuffer(uptr buffer, uptr buffer_size) { + UnmapOrDie(reinterpret_cast(buffer), buffer_size); + } + + // Releases [from, to) range of pages back to OS. + void ReleasePageRangeToOS(CompactPtrT from, CompactPtrT to) { + ReleaseMemoryPagesToOS( + allocator.CompactPtrToPointer(region_base, from), + allocator.CompactPtrToPointer(region_base, to)); + released_ranges_count++; + } + + private: + const ThisT& allocator; + const uptr region_base; + uptr released_ranges_count; + }; - // Attempts to release some RAM back to OS. The region is expected to be - // locked. - // Algorithm: - // * Sort the chunks. - // * Find ranges fully covered by free-d chunks - // * Release them to OS with madvise. + // Attempts to release RAM occupied by freed chunks back to OS. The region is + // expected to be locked. void MaybeReleaseToOS(uptr class_id) { RegionInfo *region = GetRegionInfo(class_id); const uptr chunk_size = ClassIdToSize(class_id); @@ -528,33 +792,20 @@ if (interval_ms < 0) return; - u64 now_ns = NanoTime(); - if (region->rtoi.last_release_at_ns + interval_ms * 1000000ULL > now_ns) + if (region->rtoi.last_release_at_ns + interval_ms * 1000000ULL > NanoTime()) return; // Memory was returned recently. - region->rtoi.last_release_at_ns = now_ns; - - uptr region_beg = GetRegionBeginBySizeClass(class_id); - CompactPtrT *free_array = GetFreeArray(region_beg); - SortArray(free_array, n); - const uptr scaled_chunk_size = chunk_size >> kCompactPtrScale; - const uptr kScaledGranularity = page_size >> kCompactPtrScale; + MemoryMapper memory_mapper(*this, class_id); - uptr range_beg = free_array[0]; - uptr prev = free_array[0]; - for (uptr i = 1; i < n; i++) { - uptr chunk = free_array[i]; - CHECK_GT(chunk, prev); - if (chunk - prev != scaled_chunk_size) { - CHECK_GT(chunk - prev, scaled_chunk_size); - if (prev + scaled_chunk_size - range_beg >= kScaledGranularity) { - MaybeReleaseChunkRange(region_beg, chunk_size, range_beg, prev); - region->rtoi.n_freed_at_last_release = region->stats.n_freed; - region->rtoi.num_releases++; - } - range_beg = chunk; - } - prev = chunk; + ReleaseFreeMemoryToOS( + GetFreeArray(GetRegionBeginBySizeClass(class_id)), n, chunk_size, + RoundUpTo(region->allocated_user, page_size) / page_size, + &memory_mapper); + + if (memory_mapper.GetReleasedRangesCount() > 0) { + region->rtoi.n_freed_at_last_release = region->stats.n_freed; + region->rtoi.num_releases += memory_mapper.GetReleasedRangesCount(); } + region->rtoi.last_release_at_ns = NanoTime(); } }; Index: compiler-rt/trunk/lib/sanitizer_common/tests/sanitizer_allocator_test.cc =================================================================== --- compiler-rt/trunk/lib/sanitizer_common/tests/sanitizer_allocator_test.cc +++ compiler-rt/trunk/lib/sanitizer_common/tests/sanitizer_allocator_test.cc @@ -20,6 +20,7 @@ #include "gtest/gtest.h" +#include #include #include #include @@ -1013,6 +1014,280 @@ #endif +#if SANITIZER_CAN_USE_ALLOCATOR64 + +class NoMemoryMapper { + public: + uptr last_request_buffer_size; + + NoMemoryMapper() : last_request_buffer_size(0) {} + + uptr MapPackedCounterArrayBuffer(uptr buffer_size) { + last_request_buffer_size = buffer_size; + return 0; + } + void UnmapPackedCounterArrayBuffer(uptr buffer, uptr buffer_size) {} +}; + +class RedZoneMemoryMapper { + public: + RedZoneMemoryMapper() { + const auto page_size = GetPageSize(); + buffer = MmapOrDie(3ULL * page_size, ""); + MprotectNoAccess(reinterpret_cast(buffer), page_size); + MprotectNoAccess(reinterpret_cast(buffer) + page_size * 2, page_size); + } + ~RedZoneMemoryMapper() { + UnmapOrDie(buffer, 3 * GetPageSize()); + } + + uptr MapPackedCounterArrayBuffer(uptr buffer_size) { + const auto page_size = GetPageSize(); + CHECK_EQ(buffer_size, page_size); + memset(reinterpret_cast(reinterpret_cast(buffer) + page_size), + 0, page_size); + return reinterpret_cast(buffer) + page_size; + } + void UnmapPackedCounterArrayBuffer(uptr buffer, uptr buffer_size) {} + + private: + void *buffer; +}; + +TEST(SanitizerCommon, SizeClassAllocator64PackedCounterArray) { + NoMemoryMapper no_memory_mapper; + typedef Allocator64::PackedCounterArray + NoMemoryPackedCounterArray; + + for (int i = 0; i < 64; i++) { + // Various valid counter's max values packed into one word. + NoMemoryPackedCounterArray counters_2n(1, 1ULL << i, &no_memory_mapper); + EXPECT_EQ(8ULL, no_memory_mapper.last_request_buffer_size); + + // Check the "all bit set" values too. + NoMemoryPackedCounterArray counters_2n1_1(1, ~0ULL >> i, &no_memory_mapper); + EXPECT_EQ(8ULL, no_memory_mapper.last_request_buffer_size); + + // Verify the packing ratio, the counter is expected to be packed into the + // closest power of 2 bits. + NoMemoryPackedCounterArray counters(64, 1ULL << i, &no_memory_mapper); + EXPECT_EQ(8ULL * RoundUpToPowerOfTwo(i + 1), + no_memory_mapper.last_request_buffer_size); + } + + RedZoneMemoryMapper memory_mapper; + typedef Allocator64::PackedCounterArray + RedZonePackedCounterArray; + // Go through 1, 2, 4, 8, .. 64 bits per counter. + for (int i = 0; i < 7; i++) { + // Make sure counters request one memory page for the buffer. + const u64 kNumCounters = (GetPageSize() / 8) * (64 >> i); + RedZonePackedCounterArray counters(kNumCounters, + 1ULL << ((1 << i) - 1), + &memory_mapper); + counters.Inc(0); + for (u64 c = 1; c < kNumCounters - 1; c++) { + ASSERT_EQ(0ULL, counters.Get(c)); + counters.Inc(c); + ASSERT_EQ(1ULL, counters.Get(c - 1)); + } + ASSERT_EQ(0ULL, counters.Get(kNumCounters - 1)); + counters.Inc(kNumCounters - 1); + + if (i > 0) { + counters.IncRange(0, kNumCounters - 1); + for (u64 c = 0; c < kNumCounters; c++) + ASSERT_EQ(2ULL, counters.Get(c)); + } + } +} + +class RangeRecorder { + public: + std::string reported_pages; + + RangeRecorder() + : page_size_scaled_log( + Log2(GetPageSizeCached() >> Allocator64::kCompactPtrScale)), + last_page_reported(0) {} + + void ReleasePageRangeToOS(u32 from, u32 to) { + from >>= page_size_scaled_log; + to >>= page_size_scaled_log; + ASSERT_LT(from, to); + if (!reported_pages.empty()) + ASSERT_LT(last_page_reported, from); + reported_pages.append(from - last_page_reported, '.'); + reported_pages.append(to - from, 'x'); + last_page_reported = to; + } + private: + const uptr page_size_scaled_log; + u32 last_page_reported; +}; + +TEST(SanitizerCommon, SizeClassAllocator64FreePagesRangeTracker) { + typedef Allocator64::FreePagesRangeTracker RangeTracker; + + // 'x' denotes a page to be released, '.' denotes a page to be kept around. + const char* test_cases[] = { + "", + ".", + "x", + "........", + "xxxxxxxxxxx", + "..............xxxxx", + "xxxxxxxxxxxxxxxxxx.....", + "......xxxxxxxx........", + "xxx..........xxxxxxxxxxxxxxx", + "......xxxx....xxxx........", + "xxx..........xxxxxxxx....xxxxxxx", + "x.x.x.x.x.x.x.x.x.x.x.x.", + ".x.x.x.x.x.x.x.x.x.x.x.x", + ".x.x.x.x.x.x.x.x.x.x.x.x.", + "x.x.x.x.x.x.x.x.x.x.x.x.x", + }; + + for (auto test_case : test_cases) { + RangeRecorder range_recorder; + RangeTracker tracker(&range_recorder); + for (int i = 0; test_case[i] != 0; i++) + tracker.NextPage(test_case[i] == 'x'); + tracker.Done(); + // Strip trailing '.'-pages before comparing the results as they are not + // going to be reported to range_recorder anyway. + const char* last_x = strrchr(test_case, 'x'); + std::string expected( + test_case, + last_x == nullptr ? 0 : (last_x - test_case + 1)); + EXPECT_STREQ(expected.c_str(), range_recorder.reported_pages.c_str()); + } +} + +class ReleasedPagesTrackingMemoryMapper { + public: + std::set reported_pages; + + uptr MapPackedCounterArrayBuffer(uptr buffer_size) { + reported_pages.clear(); + return reinterpret_cast(calloc(1, buffer_size)); + } + void UnmapPackedCounterArrayBuffer(uptr buffer, uptr buffer_size) { + free(reinterpret_cast(buffer)); + } + + void ReleasePageRangeToOS(u32 from, u32 to) { + uptr page_size_scaled = + GetPageSizeCached() >> Allocator64::kCompactPtrScale; + for (u32 i = from; i < to; i += page_size_scaled) + reported_pages.insert(i); + } +}; + +template +void TestReleaseFreeMemoryToOS() { + ReleasedPagesTrackingMemoryMapper memory_mapper; + const uptr kAllocatedPagesCount = 1024; + const uptr page_size = GetPageSizeCached(); + const uptr page_size_scaled = page_size >> Allocator::kCompactPtrScale; + std::mt19937 r; + uint32_t rnd_state = 42; + + for (uptr class_id = 1; class_id <= Allocator::SizeClassMapT::kLargestClassID; + class_id++) { + const uptr chunk_size = Allocator::SizeClassMapT::Size(class_id); + const uptr chunk_size_scaled = chunk_size >> Allocator::kCompactPtrScale; + const uptr max_chunks = + kAllocatedPagesCount * GetPageSizeCached() / chunk_size; + + // Generate the random free list. + std::vector free_array; + bool in_free_range = false; + uptr current_range_end = 0; + for (uptr i = 0; i < max_chunks; i++) { + if (i == current_range_end) { + in_free_range = (my_rand_r(&rnd_state) & 1U) == 1; + current_range_end += my_rand_r(&rnd_state) % 100 + 1; + } + if (in_free_range) + free_array.push_back(i * chunk_size_scaled); + } + if (free_array.empty()) + continue; + // Shuffle free_list to verify that ReleaseFreeMemoryToOS does not depend on + // the list ordering. + std::shuffle(free_array.begin(), free_array.end(), r); + + Allocator::ReleaseFreeMemoryToOS(&free_array[0], free_array.size(), + chunk_size, kAllocatedPagesCount, + &memory_mapper); + + // Verify that there are no released pages touched by used chunks and all + // ranges of free chunks big enough to contain the entire memory pages had + // these pages released. + uptr verified_released_pages = 0; + std::set free_chunks(free_array.begin(), free_array.end()); + + u32 current_chunk = 0; + in_free_range = false; + u32 current_free_range_start = 0; + for (uptr i = 0; i <= max_chunks; i++) { + bool is_free_chunk = free_chunks.find(current_chunk) != free_chunks.end(); + + if (is_free_chunk) { + if (!in_free_range) { + in_free_range = true; + current_free_range_start = current_chunk; + } + } else { + // Verify that this used chunk does not touch any released page. + for (uptr i_page = current_chunk / page_size_scaled; + i_page <= (current_chunk + chunk_size_scaled - 1) / + page_size_scaled; + i_page++) { + bool page_released = + memory_mapper.reported_pages.find(i_page * page_size_scaled) != + memory_mapper.reported_pages.end(); + ASSERT_EQ(false, page_released); + } + + if (in_free_range) { + in_free_range = false; + // Verify that all entire memory pages covered by this range of free + // chunks were released. + u32 page = RoundUpTo(current_free_range_start, page_size_scaled); + while (page + page_size_scaled <= current_chunk) { + bool page_released = + memory_mapper.reported_pages.find(page) != + memory_mapper.reported_pages.end(); + ASSERT_EQ(true, page_released); + verified_released_pages++; + page += page_size_scaled; + } + } + } + + current_chunk += chunk_size_scaled; + } + + ASSERT_EQ(memory_mapper.reported_pages.size(), verified_released_pages); + } +} + +TEST(SanitizerCommon, SizeClassAllocator64ReleaseFreeMemoryToOS) { + TestReleaseFreeMemoryToOS(); +} + +TEST(SanitizerCommon, SizeClassAllocator64CompactReleaseFreeMemoryToOS) { + TestReleaseFreeMemoryToOS(); +} + +TEST(SanitizerCommon, SizeClassAllocator64VeryCompactReleaseFreeMemoryToOS) { + TestReleaseFreeMemoryToOS(); +} + +#endif // SANITIZER_CAN_USE_ALLOCATOR64 + TEST(SanitizerCommon, TwoLevelByteMap) { const u64 kSize1 = 1 << 6, kSize2 = 1 << 12; const u64 n = kSize1 * kSize2;