Index: lib/scudo/scudo_allocator.cpp =================================================================== --- lib/scudo/scudo_allocator.cpp +++ lib/scudo/scudo_allocator.cpp @@ -226,6 +226,12 @@ bool ZeroContents; bool DeleteSizeMismatch; + bool CheckRssLimit; + uptr HardRssLimitMb; + uptr SoftRssLimitMb; + atomic_uint8_t RssLimitExceeded; + atomic_uint64_t RssLastCheckedAtNS; + explicit ScudoAllocator(LinkerInitialized) : AllocatorQuarantine(LINKER_INITIALIZED) {} @@ -270,6 +276,8 @@ SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null); BackendAllocator.init(common_flags()->allocator_release_to_os_interval_ms); + HardRssLimitMb = common_flags()->hard_rss_limit_mb; + SoftRssLimitMb = common_flags()->soft_rss_limit_mb; AllocatorQuarantine.Init( static_cast(getFlags()->QuarantineSizeKb) << 10, static_cast(getFlags()->ThreadLocalQuarantineSizeKb) << 10); @@ -280,6 +288,10 @@ GlobalPrng.init(); Cookie = GlobalPrng.getU64(); + + CheckRssLimit = HardRssLimitMb || SoftRssLimitMb; + if (CheckRssLimit) + atomic_store_relaxed(&RssLastCheckedAtNS, NanoTime()); } // Helper function that checks for a valid Scudo chunk. nullptr isn't. @@ -293,6 +305,38 @@ return getScudoChunk(UserBeg)->isValid(); } + // Opportunistic RSS limit check. This will update the RSS limit status, if + // it can, every 100ms, otherwise it will just return the current one. + bool isRssLimitExceeded() { + u64 LastCheck = atomic_load_relaxed(&RssLastCheckedAtNS); + const u64 CurrentCheck = NanoTime(); + if (LIKELY(CurrentCheck < LastCheck + (100ULL * 1000000ULL))) + return atomic_load_relaxed(&RssLimitExceeded); + if (!atomic_compare_exchange_weak(&RssLastCheckedAtNS, &LastCheck, + CurrentCheck, memory_order_relaxed)) + return atomic_load_relaxed(&RssLimitExceeded); + const uptr CurrentRssMb = GetRSS() >> 20; + if (HardRssLimitMb && HardRssLimitMb < CurrentRssMb) { + Report("%s: hard RSS limit exhausted (%zdMb vs %zdMb)\n", + SanitizerToolName, HardRssLimitMb, CurrentRssMb); + DumpProcessMap(); + Die(); + } + if (SoftRssLimitMb) { + if (atomic_load_relaxed(&RssLimitExceeded)) { + if (CurrentRssMb <= SoftRssLimitMb) + atomic_store_relaxed(&RssLimitExceeded, false); + } else { + if (CurrentRssMb > SoftRssLimitMb) { + atomic_store_relaxed(&RssLimitExceeded, true); + Report("%s: soft RSS limit exhausted (%zdMb vs %zdMb)\n", + SanitizerToolName, SoftRssLimitMb, CurrentRssMb); + } + } + } + return atomic_load_relaxed(&RssLimitExceeded); + } + // Allocates a chunk. void *allocate(uptr Size, uptr Alignment, AllocType Type, bool ForceZeroContents = false) { @@ -312,6 +356,9 @@ if (UNLIKELY(AlignedSize >= MaxAllowedMallocSize)) return FailureHandler::OnBadRequest(); + if (CheckRssLimit && UNLIKELY(isRssLimitExceeded())) + return FailureHandler::OnOOM(); + // Primary and Secondary backed allocations have a different treatment. We // deal with alignment requirements of Primary serviced allocations here, // but the Secondary will take care of its own alignment needs. Index: test/scudo/rss.c =================================================================== --- /dev/null +++ test/scudo/rss.c @@ -0,0 +1,50 @@ +// RUN: %clang_scudo %s -o %t +// RUN: %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-nolimit +// RUN: %env_scudo_opts="soft_rss_limit_mb=256" %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-nolimit +// RUN: %env_scudo_opts="hard_rss_limit_mb=256" %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-nolimit +// RUN: %env_scudo_opts="soft_rss_limit_mb=64:allocator_may_return_null=0" not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-softlimit +// RUN: %env_scudo_opts="soft_rss_limit_mb=64:allocator_may_return_null=1" %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-softlimit-returnnull +// RUN: %env_scudo_opts="hard_rss_limit_mb=64:allocator_may_return_null=0" not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-hardlimit +// RUN: %env_scudo_opts="hard_rss_limit_mb=64:allocator_may_return_null=1" not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-hardlimit + +// Tests that the soft and hard RSS limits work as intended. Without limit or +// with a high limit, the test should pass without any malloc returning NULL or +// the program dying. +// If a limit is specified, it should return some NULL or die depending on +// allocator_may_return_null. This should also work without statm. + +#include +#include +#include +#include + +static const size_t kNumAllocs = 128; +static const size_t kAllocSize = 1 << 20; // 1MB. + +static void *allocs[kNumAllocs]; + +int main(int argc, char *argv[]) { + int returned_null = 0; + for (int i = 0; i < kNumAllocs; i++) { + if ((i & 0xf) == 0) + usleep(50000); + allocs[i] = malloc(kAllocSize); + if (allocs[i]) + memset(allocs[i], 0xff, kAllocSize); // Dirty the pages. + else + returned_null++; + } + for (int i = 0; i < kNumAllocs; i++) + free(allocs[i]); + if (returned_null == 0) + printf("All malloc calls succeeded\n"); + else + printf("%d malloc calls returned NULL\n", returned_null); + return 0; +} + +// CHECK-nolimit: All malloc calls succeeded +// CHECK-softlimit: soft RSS limit exhausted +// CHECK-softlimit-returnnull: soft RSS limit exhausted +// CHECK-softlimit-returnnull: malloc calls returned NULL +// CHECK-hardlimit: hard RSS limit exhausted