diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.h b/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.h --- a/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.h @@ -25,7 +25,7 @@ public: enum class Compression : u8 { None = 0, - Test, + Delta, }; constexpr StackStore() = default; diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.cpp --- a/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.cpp @@ -10,6 +10,8 @@ #include "sanitizer_atomic.h" #include "sanitizer_common.h" +#include "sanitizer_internal_defs.h" +#include "sanitizer_leb128.h" #include "sanitizer_stacktrace.h" namespace __sanitizer { @@ -125,6 +127,42 @@ return Create(); } +static u8 *CompressDelta(const uptr *from, const uptr *from_end, u8 *to, + u8 *to_end) { + uptr prev = 0; + for (; from < from_end; ++from) { + sptr diff = *from - prev; + EncodeSLEB128(diff, [&](u8 byte) { + CHECK_LT(to, to_end); + *(to++) = byte; + }); + prev += diff; + } + return to; +} + +static uptr *UncompressDelta(const u8 *from, const u8 *from_end, uptr *to, + uptr *to_end) { + uptr prev = 0; + for (; to < to_end; ++to) { + sptr diff = DecodeSLEB128([&]() -> u8 { + CHECK_LT(from, from_end); + return *(from++); + }); + prev += diff; + *to = prev; + } + return to; +} + +namespace { +struct PackedHeader { + uptr size; + StackStore::Compression type; + u8 data[]; +}; +} // namespace + uptr *StackStore::BlockInfo::GetOrUnpack() { SpinMutexLock l(&mtx_); switch (state) { @@ -137,10 +175,34 @@ break; } - uptr *ptr = Get(); + u8 *ptr = reinterpret_cast(Get()); CHECK_NE(nullptr, ptr); - // Fake unpacking. - for (uptr i = 0; i < kBlockSizeFrames; ++i) ptr[i] ^= uptr(0) - 1; + const PackedHeader *header = reinterpret_cast(ptr); + CHECK_LE(header->size, kBlockSizeBytes); + CHECK_GE(header->size, sizeof(PackedHeader)); + + uptr packed_size_aligned = RoundUpTo(header->size, GetPageSizeCached()); + + uptr *unpacked = reinterpret_cast( + MmapNoReserveOrDie(kBlockSizeBytes, "StackStoreUnpack")); + + uptr *unpacked_end; + switch (header->type) { + case Compression::Delta: + unpacked_end = UncompressDelta(header->data, ptr + header->size, unpacked, + unpacked + kBlockSizeFrames); + break; + default: + UNREACHABLE("Unexpected type"); + break; + } + + CHECK_EQ(kBlockSizeFrames, unpacked_end - unpacked); + + MprotectReadOnly((uptr)unpacked, kBlockSizeBytes); + atomic_store(&data_, reinterpret_cast(unpacked), memory_order_release); + UnmapOrDie(ptr, packed_size_aligned); + state = State::Unpacked; return Get(); } @@ -162,17 +224,45 @@ if (!ptr || !Stored(0)) return 0; - // Fake packing. - for (uptr i = 0; i < kBlockSizeFrames; ++i) ptr[i] ^= uptr(0) - 1; + u8 *packed = reinterpret_cast( + MmapNoReserveOrDie(kBlockSizeBytes, "StackStorePack")); + PackedHeader *header = reinterpret_cast(packed); + u8 *alloc_end = reinterpret_cast(header) + kBlockSizeBytes; + + u8 *packed_end = 0; + switch (type) { + case Compression::Delta: + packed_end = + CompressDelta(ptr, ptr + kBlockSizeFrames, header->data, alloc_end); + break; + default: + UNREACHABLE("Unexpected type"); + break; + } + + header->type = type; + header->size = packed_end - packed; + + uptr packed_size_aligned = RoundUpTo(header->size, GetPageSizeCached()); + UnmapOrDie(packed + packed_size_aligned, + kBlockSizeBytes - packed_size_aligned); + MprotectReadOnly((uptr)packed, packed_size_aligned); + + atomic_store(&data_, reinterpret_cast(packed), memory_order_release); + UnmapOrDie(ptr, StackStore::kBlockSizeBytes); + state = State::Packed; - return kBlockSizeBytes - kBlockSizeBytes / 10; + return kBlockSizeBytes - packed_size_aligned; } uptr StackStore::BlockInfo::Allocated() const { SpinMutexLock l(&mtx_); switch (state) { - case State::Packed: - return kBlockSizeBytes / 10; + case State::Packed: { + const PackedHeader *ptr = reinterpret_cast(Get()); + CHECK_NE(nullptr, ptr); + return RoundUpTo(ptr->size, GetPageSizeCached()); + } case State::Unpacked: case State::Storing: return kBlockSizeBytes; diff --git a/compiler-rt/lib/sanitizer_common/tests/sanitizer_stack_store_test.cpp b/compiler-rt/lib/sanitizer_common/tests/sanitizer_stack_store_test.cpp --- a/compiler-rt/lib/sanitizer_common/tests/sanitizer_stack_store_test.cpp +++ b/compiler-rt/lib/sanitizer_common/tests/sanitizer_stack_store_test.cpp @@ -125,7 +125,7 @@ public ::testing::WithParamInterface {}; INSTANTIATE_TEST_SUITE_P(PackUnpacks, StackStorePackTest, - ::testing::ValuesIn({StackStore::Compression::Test})); + ::testing::ValuesIn({StackStore::Compression::Delta})); TEST_P(StackStorePackTest, PackUnpack) { std::vector ids;