diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.h b/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.h --- a/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.h @@ -25,7 +25,7 @@ public: enum class Compression : u8 { None = 0, - Test, + Delta, }; constexpr StackStore() = default; diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.cpp --- a/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.cpp @@ -10,6 +10,8 @@ #include "sanitizer_atomic.h" #include "sanitizer_common.h" +#include "sanitizer_internal_defs.h" +#include "sanitizer_leb128.h" #include "sanitizer_stacktrace.h" namespace __sanitizer { @@ -128,6 +130,37 @@ return Create(); } +static u8 *CompressDelta(const uptr *from, const uptr *from_end, u8 *to, + u8 *to_end) { + uptr prev = 0; + for (; from < from_end; ++from) { + sptr diff = *from - prev; + to = EncodeSLEB128(diff, to, to_end); + prev += diff; + } + return to; +} + +static uptr *UncompressDelta(const u8 *from, const u8 *from_end, uptr *to, + uptr *to_end) { + uptr prev = 0; + for (; to < to_end; ++to) { + sptr diff; + from = DecodeSLEB128(from, from_end, &diff); + prev += diff; + *to = prev; + } + return to; +} + +namespace { +struct PackedHeader { + uptr size; + StackStore::Compression type; + u8 data[]; +}; +} // namespace + uptr *StackStore::BlockInfo::GetOrUnpack() { SpinMutexLock l(&mtx_); switch (state) { @@ -140,10 +173,34 @@ break; } - uptr *ptr = Get(); + u8 *ptr = reinterpret_cast(Get()); CHECK_NE(nullptr, ptr); - // Fake unpacking. - for (uptr i = 0; i < kBlockSizeFrames; ++i) ptr[i] = ~ptr[i]; + const PackedHeader *header = reinterpret_cast(ptr); + CHECK_LE(header->size, kBlockSizeBytes); + CHECK_GE(header->size, sizeof(PackedHeader)); + + uptr packed_size_aligned = RoundUpTo(header->size, GetPageSizeCached()); + + uptr *unpacked = reinterpret_cast( + MmapNoReserveOrDie(kBlockSizeBytes, "StackStoreUnpack")); + + uptr *unpacked_end; + switch (header->type) { + case Compression::Delta: + unpacked_end = UncompressDelta(header->data, ptr + header->size, unpacked, + unpacked + kBlockSizeFrames); + break; + default: + UNREACHABLE("Unexpected type"); + break; + } + + CHECK_EQ(kBlockSizeFrames, unpacked_end - unpacked); + + MprotectReadOnly(reinterpret_cast(unpacked), kBlockSizeBytes); + atomic_store(&data_, reinterpret_cast(unpacked), memory_order_release); + UnmapOrDie(ptr, packed_size_aligned); + state = State::Unpacked; return Get(); } @@ -165,17 +222,56 @@ if (!ptr || !Stored(0)) return 0; - // Fake packing. - for (uptr i = 0; i < kBlockSizeFrames; ++i) ptr[i] = ~ptr[i]; + u8 *packed = reinterpret_cast( + MmapNoReserveOrDie(kBlockSizeBytes, "StackStorePack")); + PackedHeader *header = reinterpret_cast(packed); + u8 *alloc_end = packed + kBlockSizeBytes; + + u8 *packed_end = nullptr; + switch (type) { + case Compression::Delta: + packed_end = + CompressDelta(ptr, ptr + kBlockSizeFrames, header->data, alloc_end); + break; + default: + UNREACHABLE("Unexpected type"); + break; + } + + header->type = type; + header->size = packed_end - packed; + + VPrintf(1, "Packed block of %zu KiB to %zu KiB\n", kBlockSizeBytes >> 10, + header->size >> 10); + + if (kBlockSizeBytes - header->size < kBlockSizeBytes / 8) { + VPrintf(1, "Undo and keep block unpacked\n"); + MprotectReadOnly(reinterpret_cast(ptr), kBlockSizeBytes); + UnmapOrDie(packed, kBlockSizeBytes); + state = State::Unpacked; + return 0; + } + + uptr packed_size_aligned = RoundUpTo(header->size, GetPageSizeCached()); + UnmapOrDie(packed + packed_size_aligned, + kBlockSizeBytes - packed_size_aligned); + MprotectReadOnly(reinterpret_cast(packed), packed_size_aligned); + + atomic_store(&data_, reinterpret_cast(packed), memory_order_release); + UnmapOrDie(ptr, kBlockSizeBytes); + state = State::Packed; - return kBlockSizeBytes - kBlockSizeBytes / 10; + return kBlockSizeBytes - packed_size_aligned; } uptr StackStore::BlockInfo::Allocated() const { SpinMutexLock l(&mtx_); switch (state) { - case State::Packed: - return kBlockSizeBytes / 10; + case State::Packed: { + const PackedHeader *ptr = reinterpret_cast(Get()); + CHECK_NE(nullptr, ptr); + return RoundUpTo(ptr->size, GetPageSizeCached()); + } case State::Unpacked: case State::Storing: return kBlockSizeBytes; @@ -184,7 +280,7 @@ void StackStore::BlockInfo::TestOnlyUnmap() { if (uptr *ptr = Get()) - UnmapOrDie(ptr, StackStore::kBlockSizeBytes); + UnmapOrDie(ptr, kBlockSizeBytes); } bool StackStore::BlockInfo::Stored(uptr n) { diff --git a/compiler-rt/lib/sanitizer_common/tests/sanitizer_stack_store_test.cpp b/compiler-rt/lib/sanitizer_common/tests/sanitizer_stack_store_test.cpp --- a/compiler-rt/lib/sanitizer_common/tests/sanitizer_stack_store_test.cpp +++ b/compiler-rt/lib/sanitizer_common/tests/sanitizer_stack_store_test.cpp @@ -135,7 +135,8 @@ INSTANTIATE_TEST_SUITE_P( PackUnpacks, StackStorePackTest, ::testing::ValuesIn({ - StackStorePackTest::ParamType(StackStore::Compression::Test, 4), + StackStorePackTest::ParamType(StackStore::Compression::Delta, + FIRST_32_SECOND_64(2, 6)), })); TEST_P(StackStorePackTest, PackUnpack) { @@ -175,4 +176,23 @@ EXPECT_EQ(0u, CountPackedBlocks()); } +TEST_P(StackStorePackTest, Failed) { + MurMur2Hash64Builder h(0); + StackStore::Compression type = GetParam().first; + std::vector frames(200); + for (uptr i = 0; i < kBlockSizeFrames * 4 / frames.size(); ++i) { + for (uptr& f : frames) { + h.add(1); + // Make it difficult to pack. + f = h.get(); + } + uptr pack = 0; + store_.Store(StackTrace(frames.data(), frames.size()), &pack); + if (pack) + EXPECT_EQ(0u, store_.Pack(type)); + } + + EXPECT_EQ(0u, CountPackedBlocks()); +} + } // namespace __sanitizer