Index: lib/scudo/scudo_allocator.cpp =================================================================== --- lib/scudo/scudo_allocator.cpp +++ lib/scudo/scudo_allocator.cpp @@ -226,6 +226,12 @@ bool ZeroContents; bool DeleteSizeMismatch; + bool CheckRssLimit; + uptr HardRssLimitMb; + uptr SoftRssLimitMb; + atomic_uint8_t RssLimitExceeded; + atomic_uint64_t RssLastCheckedAtNS; + explicit ScudoAllocator(LinkerInitialized) : AllocatorQuarantine(LINKER_INITIALIZED) {} @@ -270,6 +276,8 @@ SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null); BackendAllocator.init(common_flags()->allocator_release_to_os_interval_ms); + HardRssLimitMb = common_flags()->hard_rss_limit_mb; + SoftRssLimitMb = common_flags()->soft_rss_limit_mb; AllocatorQuarantine.Init( static_cast(getFlags()->QuarantineSizeKb) << 10, static_cast(getFlags()->ThreadLocalQuarantineSizeKb) << 10); @@ -280,6 +288,10 @@ GlobalPrng.init(); Cookie = GlobalPrng.getU64(); + + CheckRssLimit = HardRssLimitMb || SoftRssLimitMb; + if (CheckRssLimit) + atomic_store_relaxed(&RssLastCheckedAtNS, NanoTime()); } // Helper function that checks for a valid Scudo chunk. nullptr isn't. @@ -293,6 +305,36 @@ return getScudoChunk(UserBeg)->isValid(); } + // Opportunistic RSS limit check. This will update the RSS limit status, if + // it can, every 100ms, otherwise it will just return the current one. + bool isRssLimitExceeded() { + u64 LastCheck = atomic_load_relaxed(&RssLastCheckedAtNS); + const u64 CurrentCheck = NanoTime(); + if (CurrentCheck < LastCheck + (100ULL * 1000000ULL)) + return atomic_load_relaxed(&RssLimitExceeded); + if (!atomic_compare_exchange_weak(&RssLastCheckedAtNS, &LastCheck, + CurrentCheck, memory_order_relaxed)) + return atomic_load_relaxed(&RssLimitExceeded); + const uptr CurrentRssMb = GetRSS() >> 20; + if (HardRssLimitMb && HardRssLimitMb < CurrentRssMb) { + Report("%s: hard RSS limit exhausted (%zdMb vs %zdMb)\n", + SanitizerToolName, HardRssLimitMb, CurrentRssMb); + DumpProcessMap(); + Die(); + } + if (SoftRssLimitMb) { + const bool ReachedSoftRssLimit = atomic_load_relaxed(&RssLimitExceeded); + if (SoftRssLimitMb < CurrentRssMb && !ReachedSoftRssLimit) { + Report("%s: soft RSS limit exhausted (%zdMb vs %zdMb)\n", + SanitizerToolName, SoftRssLimitMb, CurrentRssMb); + atomic_store_relaxed(&RssLimitExceeded, true); + } else if (SoftRssLimitMb >= CurrentRssMb && ReachedSoftRssLimit) { + atomic_store_relaxed(&RssLimitExceeded, false); + } + } + return atomic_load_relaxed(&RssLimitExceeded); + } + // Allocates a chunk. void *allocate(uptr Size, uptr Alignment, AllocType Type, bool ForceZeroContents = false) { @@ -312,6 +354,9 @@ if (UNLIKELY(AlignedSize >= MaxAllowedMallocSize)) return FailureHandler::OnBadRequest(); + if (CheckRssLimit && isRssLimitExceeded()) + return FailureHandler::OnOOM(); + // Primary and Secondary backed allocations have a different treatment. We // deal with alignment requirements of Primary serviced allocations here, // but the Secondary will take care of its own alignment needs. Index: test/scudo/rss.c =================================================================== --- /dev/null +++ test/scudo/rss.c @@ -0,0 +1,51 @@ +// RUN: %clang_scudo %s -o %t +// RUN: %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-nolimit +// RUN: %env_scudo_opts="soft_rss_limit_mb=256" %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-nolimit +// RUN: %env_scudo_opts="hard_rss_limit_mb=256" %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-nolimit +// RUN: %env_scudo_opts="soft_rss_limit_mb=64:allocator_may_return_null=0" not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-softlimit +// RUN: %env_scudo_opts="soft_rss_limit_mb=64:allocator_may_return_null=1" %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-softlimit +// RUN: %env_scudo_opts="soft_rss_limit_mb=64:allocator_may_return_null=0:can_use_proc_maps_statm=0" not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-softlimit +// RUN: %env_scudo_opts="soft_rss_limit_mb=64:allocator_may_return_null=1:can_use_proc_maps_statm=0" %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-softlimit +// RUN: %env_scudo_opts="hard_rss_limit_mb=64:allocator_may_return_null=0" not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-hardlimit +// RUN: %env_scudo_opts="hard_rss_limit_mb=64:allocator_may_return_null=1" not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-hardlimit +// RUN: %env_scudo_opts="hard_rss_limit_mb=64:allocator_may_return_null=0:can_use_proc_maps_statm=0" not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-hardlimit +// RUN: %env_scudo_opts="hard_rss_limit_mb=64:allocator_may_return_null=1:can_use_proc_maps_statm=0" not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-hardlimit + +// Tests that the soft and hard RSS limits work as intended. Without limit or +// with a high limit, the test should pass without any malloc returning NULL or +// the program dying. +// If a limit is specified, it should return some NULL or die depending on +// allocator_may_return_null. This should also work without statm. + +#include +#include +#include +#include + +static const size_t kNumAllocs = 128; +static const size_t kAllocSize = 1 << 20; // 1MB. + +static void *allocs[kNumAllocs]; + +int main(int argc, char *argv[]) { + int returned_null = 0; + for (int i = 0; i < kNumAllocs; i++) { + if ((i & 0xf) == 0) + usleep(50000); + allocs[i] = malloc(kAllocSize); + if (allocs[i]) + memset(allocs[i], 0xff, kAllocSize); // Dirty the pages. + else + returned_null++; + } + for (int i = 0; i < kNumAllocs; i++) + free(allocs[i]); + if (returned_null) + printf("Some of the malloc calls returned NULL (%d)\n", returned_null); + printf("Done\n"); + return 0; +} + +// CHECK-nolimit-NOT: Some of the malloc calls returned NULL +// CHECK-softlimit: soft RSS limit exhausted +// CHECK-hardlimit: hard RSS limit exhausted