scudo: Add way to set release interval for both allocators.

[cferris]

Summary

For Android, set the defaults so that the primary is 1 second, but
the secondary is set to always release.

Add the ability to set these release intervals in mallopt.

[cferris]

Updated mallopt for android.

[cferris]

Removed the minimum flags and create and Android specific mallopt setting.

[cryptoad]

The idea sounds good, but as discussed by mail I'd like to avoid breaking the old option.
As pcc@ suggested, it might be time to move the release interval into the allocator config rather than have them be flags, since now there is a way to influence them via mallopt.

[cferris]

Changed this enough, so I'm abandoning this.

compiler-rt/lib/scudo/standalone/combined.h

 #include "gwp_asan/optional/backtrace.h"
 #include "gwp_asan/optional/segv_handler.h"
 #endif // GWP_ASAN_HOOKS
 
 extern "C" inline void EmptyCallback() {}
 
 namespace scudo {
 
+enum class Option { PrimaryReleaseInterval, SecondaryReleaseInterval };
+
 template <class Params, void (*PostInitCallback)(void) = EmptyCallback>
 class Allocator {
 public:
   using PrimaryT = typename Params::Primary;
   using CacheT = typename PrimaryT::CacheT;
   typedef Allocator<Params, PostInitCallback> ThisT;
   typedef typename Params::template TSDRegistryT<ThisT> TSDRegistryT;
 
void initLinkerInitialized() {
     Options.MayReturnNull = getFlags()->may_return_null;
     Options.ZeroContents = getFlags()->zero_contents;
     Options.DeallocTypeMismatch = getFlags()->dealloc_type_mismatch;
     Options.DeleteSizeMismatch = getFlags()->delete_size_mismatch;
     Options.QuarantineMaxChunkSize =
         static_cast<u32>(getFlags()->quarantine_max_chunk_size);
 
     Stats.initLinkerInitialized();
-    const s32 ReleaseToOsIntervalMs = getFlags()->release_to_os_interval_ms;
-    Primary.initLinkerInitialized(ReleaseToOsIntervalMs);
-    Secondary.initLinkerInitialized(&Stats, ReleaseToOsIntervalMs);
+    Primary.initLinkerInitialized(getFlags()->primary_release_to_os_interval_ms);
+    Secondary.initLinkerInitialized(&Stats, getFlags()->secondary_release_to_os_interval_ms);
 
     Quarantine.init(
         static_cast<uptr>(getFlags()->quarantine_size_kb << 10),
         static_cast<uptr>(getFlags()->thread_local_quarantine_size_kb << 10));
   }
 
   // Initialize the embedded GWP-ASan instance. Requires the main allocator to
   // be functional, best called from PostInitCallback.

 #endif
   }
 
   bool canReturnNull() {
     initThreadMaybe();
     return Options.MayReturnNull;
   }
 
-  // TODO(kostyak): implement this as a "backend" to mallopt.
-  bool setOption(UNUSED uptr Option, UNUSED uptr Value) { return false; }
+  bool setOption(Option O, sptr Value) {
+    if (O == Option::PrimaryReleaseInterval) {
+      Primary.setReleaseToOsIntervalMs(Value);
+      return true;
+    } else if (O == Option::SecondaryReleaseInterval) {
+      Secondary.setReleaseToOsIntervalMs(Value);
+      return true;
+    }
+    return false;
+  }
 
   // Return the usable size for a given chunk. Technically we lie, as we just
   // report the actual size of a chunk. This is done to counteract code actively
   // writing past the end of a chunk (like sqlite3) when the usable size allows
   // for it, which then forces realloc to copy the usable size of a chunk as
   // opposed to its actual size.
   uptr getUsableSize(const void *Ptr) {
     initThreadMaybe();

compiler-rt/lib/scudo/standalone/flags.inc

SCUDO_FLAG(int, rss_limit_mb, -1,
            "attempted past that limit (depending on may_return_null). Negative "
            "values disable the feature.")
 
 SCUDO_FLAG(bool, may_return_null, true,
            "Indicate whether the allocator should terminate instead of "
            "returning NULL in otherwise non-fatal error scenarios, eg: OOM, "
            "invalid allocation alignments, etc.")
 
-SCUDO_FLAG(int, release_to_os_interval_ms, SCUDO_ANDROID ? 1000 : 5000,
+SCUDO_FLAG(int, primary_release_to_os_interval_ms, SCUDO_ANDROID ? 1000 : 5000,
            "Interval (in milliseconds) at which to attempt release of unused "
-           "memory to the OS. Negative values disable the feature.")
+           "memory to the OS in the primary. Negative values disable the "
+           "feature.")
+
+SCUDO_FLAG(int, secondary_release_to_os_interval_ms, SCUDO_ANDROID ? 0 : 5000,
+           "Interval (in milliseconds) at which to attempt release of unused "
+           "memory to the OS in the secondary. Negative values disable the "
+           "feature.")

compiler-rt/lib/scudo/standalone/primary32.h

void initLinkerInitialized(s32 ReleaseToOsInterval) {
     const uptr PageSize = getPageSizeCached();
     for (uptr I = 0; I < NumClasses; I++) {
       SizeClassInfo *Sci = getSizeClassInfo(I);
       Sci->RandState = getRandomU32(&Seed);
       // See comment in the 64-bit primary about releasing smaller size classes.
       Sci->CanRelease = (I != SizeClassMap::BatchClassId) &&
                         (getSizeByClassId(I) >= (PageSize / 32));
     }
-    ReleaseToOsIntervalMs = ReleaseToOsInterval;
+    setReleaseToOsIntervalMs(ReleaseToOsInterval);
   }
   void init(s32 ReleaseToOsInterval) {
     memset(this, 0, sizeof(*this));
     initLinkerInitialized(ReleaseToOsInterval);
   }
 
   void unmapTestOnly() {
     while (NumberOfStashedRegions > 0)
void getStats(ScopedString *Str) {
     }
     Str->append("Stats: SizeClassAllocator32: %zuM mapped in %zu allocations; "
                 "remains %zu\n",
                 TotalMapped >> 20, PoppedBlocks, PoppedBlocks - PushedBlocks);
     for (uptr I = 0; I < NumClasses; I++)
       getStats(Str, I, 0);
   }
 
+  void setReleaseToOsIntervalMs(s32 Interval) {
+    atomic_store(&ReleaseToOsIntervalMs, Interval, memory_order_relaxed);
+  }
+
   uptr releaseToOS() {
     uptr TotalReleasedBytes = 0;
     for (uptr I = 0; I < NumClasses; I++) {
       SizeClassInfo *Sci = getSizeClassInfo(I);
       ScopedLock L(Sci->Mutex);
       TotalReleasedBytes += releaseToOSMaybe(Sci, I, /*Force=*/true);
     }
     return TotalReleasedBytes;
void getStats(ScopedString *Str, uptr ClassId, uptr Rss) {
     const uptr AvailableChunks = Sci->AllocatedUser / getSizeByClassId(ClassId);
     Str->append("  %02zu (%6zu): mapped: %6zuK popped: %7zu pushed: %7zu "
                 "inuse: %6zu avail: %6zu rss: %6zuK\n",
                 ClassId, getSizeByClassId(ClassId), Sci->AllocatedUser >> 10,
                 Sci->Stats.PoppedBlocks, Sci->Stats.PushedBlocks, InUse,
                 AvailableChunks, Rss >> 10);
   }
 
+  s32 getReleaseToOsIntervalMs() {
+    return atomic_load(&ReleaseToOsIntervalMs, memory_order_relaxed);
+  }
+
   NOINLINE uptr releaseToOSMaybe(SizeClassInfo *Sci, uptr ClassId,
                                  bool Force = false) {
     const uptr BlockSize = getSizeByClassId(ClassId);
     const uptr PageSize = getPageSizeCached();
 
     CHECK_GE(Sci->Stats.PoppedBlocks, Sci->Stats.PushedBlocks);
     const uptr BytesInFreeList =
         Sci->AllocatedUser -
         (Sci->Stats.PoppedBlocks - Sci->Stats.PushedBlocks) * BlockSize;
     if (BytesInFreeList < PageSize)
       return 0; // No chance to release anything.
     if ((Sci->Stats.PushedBlocks - Sci->ReleaseInfo.PushedBlocksAtLastRelease) *
             BlockSize <
         PageSize) {
       return 0; // Nothing new to release.
     }
 
     if (!Force) {
-      const s32 IntervalMs = ReleaseToOsIntervalMs;
+      const s32 IntervalMs = getReleaseToOsIntervalMs();
       if (IntervalMs < 0)
         return 0;
       if (Sci->ReleaseInfo.LastReleaseAtNs +
               static_cast<u64>(IntervalMs) * 1000000 >
           getMonotonicTime()) {
         return 0; // Memory was returned recently.
       }
     }
private:
 
   SizeClassInfo SizeClassInfoArray[NumClasses];
 
   ByteMap PossibleRegions;
   // Keep track of the lowest & highest regions allocated to avoid looping
   // through the whole NumRegions.
   uptr MinRegionIndex;
   uptr MaxRegionIndex;
-  s32 ReleaseToOsIntervalMs;
+  atomic_s32 ReleaseToOsIntervalMs;
   // Unless several threads request regions simultaneously from different size
   // classes, the stash rarely contains more than 1 entry.
   static constexpr uptr MaxStashedRegions = 4;
   HybridMutex RegionsStashMutex;
   uptr NumberOfStashedRegions;
   uptr RegionsStash[MaxStashedRegions];
 };
 
 } // namespace scudo
 
 #endif // SCUDO_PRIMARY32_H_

compiler-rt/lib/scudo/standalone/primary64.h

for (uptr I = 0; I < NumClasses; I++) {
       // On top of this, attempting to release those require more iterations and
       // memory accesses which ends up being fairly costly. The current lower
       // limit is mostly arbitrary and based on empirical observations.
       // TODO(kostyak): make the lower limit a runtime option
       Region->CanRelease = (I != SizeClassMap::BatchClassId) &&
                            (getSizeByClassId(I) >= (PageSize / 32));
       Region->RandState = getRandomU32(&Seed);
     }
-    ReleaseToOsIntervalMs = ReleaseToOsInterval;
+    setReleaseToOsIntervalMs(ReleaseToOsInterval);
 
     if (SupportsMemoryTagging)
       UseMemoryTagging = systemSupportsMemoryTagging();
   }
   void init(s32 ReleaseToOsInterval) {
     memset(this, 0, sizeof(*this));
     initLinkerInitialized(ReleaseToOsInterval);
   }
Str->append("Stats: SizeClassAllocator64: %zuM mapped (%zuM rss) in %zu "
                 "allocations; remains %zu\n",
                 TotalMapped >> 20, 0, PoppedBlocks,
                 PoppedBlocks - PushedBlocks);
 
     for (uptr I = 0; I < NumClasses; I++)
       getStats(Str, I, 0);
   }
 
+  void setReleaseToOsIntervalMs(s32 Interval) {
+    atomic_store(&ReleaseToOsIntervalMs, Interval, memory_order_relaxed);
+  }
+
   uptr releaseToOS() {
     uptr TotalReleasedBytes = 0;
     for (uptr I = 0; I < NumClasses; I++) {
       RegionInfo *Region = getRegionInfo(I);
       ScopedLock L(Region->Mutex);
       TotalReleasedBytes += releaseToOSMaybe(Region, I, /*Force=*/true);
     }
     return TotalReleasedBytes;
struct ALIGNED(SCUDO_CACHE_LINE_SIZE) RegionInfo {
     MapPlatformData Data;
     ReleaseToOsInfo ReleaseInfo;
   };
   static_assert(sizeof(RegionInfo) % SCUDO_CACHE_LINE_SIZE == 0, "");
 
   uptr PrimaryBase;
   RegionInfo *RegionInfoArray;
   MapPlatformData Data;
-  s32 ReleaseToOsIntervalMs;
+  atomic_s32 ReleaseToOsIntervalMs;
   bool UseMemoryTagging;
 
   RegionInfo *getRegionInfo(uptr ClassId) const {
     DCHECK_LT(ClassId, NumClasses);
     return &RegionInfoArray[ClassId];
   }
 
   uptr getRegionBaseByClassId(uptr ClassId) const {
Str->append("%s %02zu (%6zu): mapped: %6zuK popped: %7zu pushed: %7zu "
                 Region->Exhausted ? "F" : " ", ClassId,
                 getSizeByClassId(ClassId), Region->MappedUser >> 10,
                 Region->Stats.PoppedBlocks, Region->Stats.PushedBlocks, InUse,
                 TotalChunks, Rss >> 10, Region->ReleaseInfo.RangesReleased,
                 Region->ReleaseInfo.LastReleasedBytes >> 10, Region->RegionBeg,
                 getRegionBaseByClassId(ClassId));
   }
 
+  s32 getReleaseToOsIntervalMs() {
+    return atomic_load(&ReleaseToOsIntervalMs, memory_order_relaxed);
+  }
+
   NOINLINE uptr releaseToOSMaybe(RegionInfo *Region, uptr ClassId,
                                  bool Force = false) {
     const uptr BlockSize = getSizeByClassId(ClassId);
     const uptr PageSize = getPageSizeCached();
 
     CHECK_GE(Region->Stats.PoppedBlocks, Region->Stats.PushedBlocks);
     const uptr BytesInFreeList =
         Region->AllocatedUser -
         (Region->Stats.PoppedBlocks - Region->Stats.PushedBlocks) * BlockSize;
     if (BytesInFreeList < PageSize)
       return 0; // No chance to release anything.
     if ((Region->Stats.PushedBlocks -
          Region->ReleaseInfo.PushedBlocksAtLastRelease) *
             BlockSize <
         PageSize) {
       return 0; // Nothing new to release.
     }
 
     if (!Force) {
-      const s32 IntervalMs = ReleaseToOsIntervalMs;
+      const s32 IntervalMs = getReleaseToOsIntervalMs();
       if (IntervalMs < 0)
         return 0;
       if (Region->ReleaseInfo.LastReleaseAtNs +
               static_cast<u64>(IntervalMs) * 1000000 >
           getMonotonicTime()) {
         return 0; // Memory was returned recently.
       }
     }

compiler-rt/lib/scudo/standalone/secondary.h

 class MapAllocatorCache {
 public:
   // Fuchsia doesn't allow releasing Secondary blocks yet. Note that 0 length
   // arrays are an extension for some compilers.
   // FIXME(kostyak): support (partially) the cache on Fuchsia.
   static_assert(!SCUDO_FUCHSIA || MaxEntriesCount == 0U, "");
 
   void initLinkerInitialized(s32 ReleaseToOsInterval) {
-    ReleaseToOsIntervalMs = ReleaseToOsInterval;
+    setReleaseToOsIntervalMs(ReleaseToOsInterval);
   }
   void init(s32 ReleaseToOsInterval) {
     memset(this, 0, sizeof(*this));
     initLinkerInitialized(ReleaseToOsInterval);
   }
 
   bool store(LargeBlock::Header *H) {
     bool EntryCached = false;
const u64 Time = getMonotonicTime();
           Entries[0].Data = H->Data;
           Entries[0].Time = Time;
           EntriesCount++;
           EntryCached = true;
           break;
         }
       }
     }
+    s32 Interval;
     if (EmptyCache)
       empty();
-    else if (ReleaseToOsIntervalMs >= 0)
-      releaseOlderThan(Time -
-                       static_cast<u64>(ReleaseToOsIntervalMs) * 1000000);
+    else if ((Interval = getReleaseToOsIntervalMs()) >= 0)
+      releaseOlderThan(Time - static_cast<u64>(Interval) * 1000000);
     return EntryCached;
   }
 
   bool retrieve(uptr Size, LargeBlock::Header **H) {
     const uptr PageSize = getPageSizeCached();
     ScopedLock L(Mutex);
     if (EntriesCount == 0)
       return false;
bool retrieve(uptr Size, LargeBlock::Header **H) {
     }
     return false;
   }
 
   static bool canCache(uptr Size) {
     return MaxEntriesCount != 0U && Size <= MaxEntrySize;
   }
 
+  void setReleaseToOsIntervalMs(s32 Interval) {
+    atomic_store(&ReleaseToOsIntervalMs, Interval, memory_order_relaxed);
+  }
+
   void releaseToOS() { releaseOlderThan(UINT64_MAX); }
 
   void disable() { Mutex.lock(); }
 
   void enable() { Mutex.unlock(); }
 
 private:
   void empty() {
for (uptr I = 0; I < MaxEntriesCount; I++) {
         continue;
       releasePagesToOS(Entries[I].Block, 0,
                        Entries[I].BlockEnd - Entries[I].Block,
                        &Entries[I].Data);
       Entries[I].Time = 0;
     }
   }
 
+  s32 getReleaseToOsIntervalMs() {
+    return atomic_load(&ReleaseToOsIntervalMs, memory_order_relaxed);
+  }
+
   struct CachedBlock {
     uptr Block;
     uptr BlockEnd;
     uptr MapBase;
     uptr MapSize;
     MapPlatformData Data;
     u64 Time;
   };
 
   HybridMutex Mutex;
   CachedBlock Entries[MaxEntriesCount];
   u32 EntriesCount;
   uptr LargestSize;
   u32 IsFullEvents;
-  s32 ReleaseToOsIntervalMs;
+  atomic_s32 ReleaseToOsIntervalMs;
 };
 
 template <class CacheT> class MapAllocator {
 public:
   void initLinkerInitialized(GlobalStats *S, s32 ReleaseToOsInterval = -1) {
     Cache.initLinkerInitialized(ReleaseToOsInterval);
     Stats.initLinkerInitialized();
     if (LIKELY(S))
public:
 
   template <typename F> void iterateOverBlocks(F Callback) const {
     for (const auto &H : InUseBlocks)
       Callback(reinterpret_cast<uptr>(&H) + LargeBlock::getHeaderSize());
   }
 
   static uptr canCache(uptr Size) { return CacheT::canCache(Size); }
 
+  void setReleaseToOsIntervalMs(s32 Interval) {
+    Cache.setReleaseToOsIntervalMs(Interval);
+  }
+
   void releaseToOS() { Cache.releaseToOS(); }
 
 private:
   CacheT Cache;
 
   HybridMutex Mutex;
   DoublyLinkedList<LargeBlock::Header> InUseBlocks;
   uptr AllocatedBytes;

compiler-rt/lib/scudo/standalone/wrappers_c.inc

 }
 
 void SCUDO_PREFIX(malloc_postinit)() {
   SCUDO_ALLOCATOR.initGwpAsan();
   pthread_atfork(SCUDO_PREFIX(malloc_disable), SCUDO_PREFIX(malloc_enable),
                  SCUDO_PREFIX(malloc_enable));
 }
 
-INTERFACE WEAK int SCUDO_PREFIX(mallopt)(int param, UNUSED int value) {
+INTERFACE WEAK int SCUDO_PREFIX(mallopt)(int param, int value) {
   if (param == M_DECAY_TIME) {
-    // TODO(kostyak): set release_to_os_interval_ms accordingly.
+    int secondary_interval;
+    if (SCUDO_ANDROID) {
+      if (value == 0) {
+        // Never go below 1 second for the primary.
+        value = 1000;
+        secondary_interval = 0;
+      } else {
+        value = 1000;
+        secondary_interval = 1000;
+      }
+    } else {
+      secondary_interval = value;
+    }
+
+    SCUDO_ALLOCATOR.setOption(scudo::Option::PrimaryReleaseInterval,
+                              static_cast<scudo::sptr>(value));
+    SCUDO_ALLOCATOR.setOption(scudo::Option::SecondaryReleaseInterval,
+                              static_cast<scudo::sptr>(secondary_interval));
     return 1;
   } else if (param == M_PURGE) {
     SCUDO_ALLOCATOR.releaseToOS();
     return 1;
   }
   return 0;
 }