scudo: Make it thread-safe to set some runtime configuration flags.

[pcc]

Summary

Move some of the flags previously in Options, as well as the
UseMemoryTagging flag previously in the primary allocator, into an
atomic variable so that it can be updated while other threads are
running. Relaxed accesses are used because we only have the requirement
that the other threads see the new value eventually.

The code is set up so that the variable is generally loaded once per
allocation function call with the exception of some rarely used code
such as error handlers. The flag bits can generally stay in a register
during the execution of the allocation function which means that they
can be branched on with minimal overhead (e.g. TBZ on aarch64).

[cryptoad]

LGTM with comment

compiler-rt/lib/scudo/standalone/options.h

1

LLVM header is missing
Also I used a #ifdef construct in all the other .h, so maybe keep that or consistency? Or eventually move them all to a pragma if it works everywhere.

[pcc]

compiler-rt/lib/scudo/standalone/options.h

1

Oops, this was meant to start like all the other files (I used pragma once to start with out of laziness, then forgot to fix it before uploading). Fixed now.

I'm not actually sure why we don't use pragma once in LLVM since it's supported by all of the compilers that we support. I guess it may be something like "don't make it gratuitously difficult to build with a compiler that doesn't support it". At any rate I'll leave the situation as is for now.

[eugenis]

compiler-rt/lib/scudo/standalone/options.h

67

This could be more readable in the form of

do { ... } while (!atomic_compare_exchange_strong

Also, there is no need to reload Opts on each iteration - compare_exchange does it for you.

[pcc]

compiler-rt/lib/scudo/standalone/options.h

67

D88747

compiler-rt/lib/scudo/standalone/atomic_helpers.h

 template <typename T>
 inline typename T::Type atomic_fetch_sub(volatile T *A, typename T::Type V,
                                          memory_order MO) {
   DCHECK(!(reinterpret_cast<uptr>(A) % sizeof(*A)));
   return __atomic_fetch_sub(&A->ValDoNotUse, V, MO);
 }
 
 template <typename T>
+inline typename T::Type atomic_fetch_and(volatile T *A, typename T::Type V,
+                                         memory_order MO) {
+  DCHECK(!(reinterpret_cast<uptr>(A) % sizeof(*A)));
+  return __atomic_fetch_and(&A->ValDoNotUse, V, MO);
+}
+
+template <typename T>
+inline typename T::Type atomic_fetch_or(volatile T *A, typename T::Type V,
+                                        memory_order MO) {
+  DCHECK(!(reinterpret_cast<uptr>(A) % sizeof(*A)));
+  return __atomic_fetch_or(&A->ValDoNotUse, V, MO);
+}
+
+template <typename T>
 inline typename T::Type atomic_exchange(volatile T *A, typename T::Type V,
                                         memory_order MO) {
   DCHECK(!(reinterpret_cast<uptr>(A) % sizeof(*A)));
   typename T::Type R;
   __atomic_exchange(&A->ValDoNotUse, &V, &R, MO);
   return R;
 }
 

compiler-rt/lib/scudo/standalone/combined.h

 #define SCUDO_COMBINED_H_
 
 #include "chunk.h"
 #include "common.h"
 #include "flags.h"
 #include "flags_parser.h"
 #include "local_cache.h"
 #include "memtag.h"
+#include "options.h"
 #include "quarantine.h"
 #include "report.h"
 #include "secondary.h"
 #include "stack_depot.h"
 #include "string_utils.h"
 #include "tsd.h"
 
 #include "scudo/interface.h"
void initLinkerInitialized() {
     if (UNLIKELY(!getRandom(&Cookie, sizeof(Cookie))))
       Cookie = static_cast<u32>(getMonotonicTime() ^
                                 (reinterpret_cast<uptr>(this) >> 4));
 
     initFlags();
     reportUnrecognizedFlags();
 
     // Store some flags locally.
-    Options.MayReturnNull = getFlags()->may_return_null;
-    Options.FillContents =
-        getFlags()->zero_contents
-            ? ZeroFill
-            : (getFlags()->pattern_fill_contents ? PatternOrZeroFill : NoFill);
-    Options.DeallocTypeMismatch = getFlags()->dealloc_type_mismatch;
-    Options.DeleteSizeMismatch = getFlags()->delete_size_mismatch;
-    Options.TrackAllocationStacks = false;
-    Options.UseOddEvenTags = true;
-    Options.QuarantineMaxChunkSize =
+    if (getFlags()->may_return_null)
+      Primary.Options.set(OptionBit::MayReturnNull);
+    if (getFlags()->zero_contents)
+      Primary.Options.setFillContentsMode(ZeroFill);
+    else if (getFlags()->pattern_fill_contents)
+      Primary.Options.setFillContentsMode(PatternOrZeroFill);
+    if (getFlags()->dealloc_type_mismatch)
+      Primary.Options.set(OptionBit::DeallocTypeMismatch);
+    if (getFlags()->delete_size_mismatch)
+      Primary.Options.set(OptionBit::DeleteSizeMismatch);
+    Primary.Options.set(OptionBit::UseOddEvenTags);
+
+    QuarantineMaxChunkSize =
         static_cast<u32>(getFlags()->quarantine_max_chunk_size);
 
     Stats.initLinkerInitialized();
     const s32 ReleaseToOsIntervalMs = getFlags()->release_to_os_interval_ms;
     Primary.initLinkerInitialized(ReleaseToOsIntervalMs);
     Secondary.initLinkerInitialized(&Stats, ReleaseToOsIntervalMs);
 
     Quarantine.init(
uptr Size =
         android_unsafe_frame_pointer_chase(Stack, MaxTraceSize + DiscardFrames);
     Size = Min<uptr>(Size, MaxTraceSize + DiscardFrames);
     return Depot.insert(Stack + Min<uptr>(DiscardFrames, Size), Stack + Size);
 #else
     return 0;
 #endif
   }
 
-  uptr computeOddEvenMaskForPointerMaybe(uptr Ptr, uptr Size) {
-    if (!Options.UseOddEvenTags)
+  uptr computeOddEvenMaskForPointerMaybe(Options Options, uptr Ptr, uptr Size) {
+    if (!Options.get(OptionBit::UseOddEvenTags))
       return 0;
 
     // If a chunk's tag is odd, we want the tags of the surrounding blocks to be
     // even, and vice versa. Blocks are laid out Size bytes apart, and adding
     // Size to Ptr will flip the least significant set bit of Size in Ptr, so
     // that bit will have the pattern 010101... for consecutive blocks, which we
     // can use to determine which tag mask to use.
     return (Ptr & (1ULL << getLeastSignificantSetBitIndex(Size))) ? 0xaaaa
                                                                   : 0x5555;
   }
 
   NOINLINE void *allocate(uptr Size, Chunk::Origin Origin,
                           uptr Alignment = MinAlignment,
                           bool ZeroContents = false) {
     initThreadMaybe();
+    Options Options = Primary.Options.load();
 
 #ifdef GWP_ASAN_HOOKS
     if (UNLIKELY(GuardedAlloc.shouldSample())) {
       if (void *Ptr = GuardedAlloc.allocate(roundUpTo(Size, Alignment)))
         return Ptr;
     }
 #endif // GWP_ASAN_HOOKS
 
     const FillContentsMode FillContents = ZeroContents ? ZeroFill
                                           : TSDRegistry.getDisableMemInit()
                                               ? NoFill
-                                              : Options.FillContents;
+                                              : Options.getFillContentsMode();
 
     if (UNLIKELY(Alignment > MaxAlignment)) {
-      if (Options.MayReturnNull)
+      if (Options.get(OptionBit::MayReturnNull))
         return nullptr;
       reportAlignmentTooBig(Alignment, MaxAlignment);
     }
     if (UNLIKELY(Alignment < MinAlignment))
       Alignment = MinAlignment;
 
     // If the requested size happens to be 0 (more common than you might think),
     // allocate MinAlignment bytes on top of the header. Then add the extra
     // bytes required to fulfill the alignment requirements: we allocate enough
     // to be sure that there will be an address in the block that will satisfy
     // the alignment.
     const uptr NeededSize =
         roundUpTo(Size, MinAlignment) +
         ((Alignment > MinAlignment) ? Alignment : Chunk::getHeaderSize());
 
     // Takes care of extravagantly large sizes as well as integer overflows.
     static_assert(MaxAllowedMallocSize < UINTPTR_MAX - MaxAlignment, "");
     if (UNLIKELY(Size >= MaxAllowedMallocSize)) {
-      if (Options.MayReturnNull)
+      if (Options.get(OptionBit::MayReturnNull))
         return nullptr;
       reportAllocationSizeTooBig(Size, NeededSize, MaxAllowedMallocSize);
     }
     DCHECK_LE(Size, NeededSize);
 
     void *Block = nullptr;
     uptr ClassId = 0;
     uptr SecondaryBlockEnd = 0;
if (LIKELY(PrimaryT::canAllocate(NeededSize))) {
       if (UnlockRequired)
         TSD->unlock();
     }
     if (UNLIKELY(ClassId == 0))
       Block = Secondary.allocate(NeededSize, Alignment, &SecondaryBlockEnd,
                                  FillContents);
 
     if (UNLIKELY(!Block)) {
-      if (Options.MayReturnNull)
+      if (Options.get(OptionBit::MayReturnNull))
         return nullptr;
       reportOutOfMemory(NeededSize);
     }
 
     const uptr BlockUptr = reinterpret_cast<uptr>(Block);
     const uptr UnalignedUserPtr = BlockUptr + Chunk::getHeaderSize();
     const uptr UserPtr = roundUpTo(UnalignedUserPtr, Alignment);
 
     void *Ptr = reinterpret_cast<void *>(UserPtr);
     void *TaggedPtr = Ptr;
     if (LIKELY(ClassId)) {
       // We only need to zero or tag the contents for Primary backed
       // allocations. We only set tags for primary allocations in order to avoid
       // faulting potentially large numbers of pages for large secondary
       // allocations. We assume that guard pages are enough to protect these
       // allocations.
       //
       // FIXME: When the kernel provides a way to set the background tag of a
       // mapping, we should be able to tag secondary allocations as well.
       //
       // When memory tagging is enabled, zeroing the contents is done as part of
       // setting the tag.
-      if (UNLIKELY(useMemoryTagging())) {
+      if (UNLIKELY(useMemoryTagging(Options))) {
         uptr PrevUserPtr;
         Chunk::UnpackedHeader Header;
         const uptr BlockSize = PrimaryT::getSizeByClassId(ClassId);
         const uptr BlockEnd = BlockUptr + BlockSize;
         // If possible, try to reuse the UAF tag that was set by deallocate().
         // For simplicity, only reuse tags if we have the same start address as
         // the previous allocation. This handles the majority of cases since
         // most allocations will not be more aligned than the minimum alignment.
if (LIKELY(ClassId)) {
                                        archMemoryTagGranuleSize())));
           } else if (Size) {
             // Clear any stack metadata that may have previously been stored in
             // the chunk data.
             memset(TaggedPtr, 0, archMemoryTagGranuleSize());
           }
         } else {
           const uptr OddEvenMask =
-              computeOddEvenMaskForPointerMaybe(BlockUptr, BlockSize);
+              computeOddEvenMaskForPointerMaybe(Options, BlockUptr, BlockSize);
           TaggedPtr = prepareTaggedChunk(Ptr, Size, OddEvenMask, BlockEnd);
         }
-        storeAllocationStackMaybe(Ptr);
+        storeAllocationStackMaybe(Options, Ptr);
       } else if (UNLIKELY(FillContents != NoFill)) {
         // This condition is not necessarily unlikely, but since memset is
         // costly, we might as well mark it as such.
         memset(Block, FillContents == ZeroFill ? 0 : PatternFillByte,
                PrimaryT::getSizeByClassId(ClassId));
       }
     }
 
NOINLINE void deallocate(void *Ptr, Chunk::Origin Origin, uptr DeleteSize = 0,
                            UNUSED uptr Alignment = MinAlignment) {
     // For a deallocation, we only ensure minimal initialization, meaning thread
     // local data will be left uninitialized for now (when using ELF TLS). The
     // fallback cache will be used instead. This is a workaround for a situation
     // where the only heap operation performed in a thread would be a free past
     // the TLS destructors, ending up in initialized thread specific data never
     // being destroyed properly. Any other heap operation will do a full init.
     initThreadMaybe(/*MinimalInit=*/true);
+    Options Options = Primary.Options.load();
 
 #ifdef GWP_ASAN_HOOKS
     if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr))) {
       GuardedAlloc.deallocate(Ptr);
       return;
     }
 #endif // GWP_ASAN_HOOKS
 
     if (&__scudo_deallocate_hook)
       __scudo_deallocate_hook(Ptr);
 
     if (UNLIKELY(!Ptr))
       return;
     if (UNLIKELY(!isAligned(reinterpret_cast<uptr>(Ptr), MinAlignment)))
       reportMisalignedPointer(AllocatorAction::Deallocating, Ptr);
 
     Ptr = untagPointerMaybe(Ptr);
 
     Chunk::UnpackedHeader Header;
     Chunk::loadHeader(Cookie, Ptr, &Header);
 
     if (UNLIKELY(Header.State != Chunk::State::Allocated))
       reportInvalidChunkState(AllocatorAction::Deallocating, Ptr);
-    if (Options.DeallocTypeMismatch) {
+    if (Options.get(OptionBit::DeallocTypeMismatch)) {
       if (Header.OriginOrWasZeroed != Origin) {
         // With the exception of memalign'd chunks, that can be still be free'd.
         if (UNLIKELY(Header.OriginOrWasZeroed != Chunk::Origin::Memalign ||
                      Origin != Chunk::Origin::Malloc))
           reportDeallocTypeMismatch(AllocatorAction::Deallocating, Ptr,
                                     Header.OriginOrWasZeroed, Origin);
       }
     }
 
     const uptr Size = getSize(Ptr, &Header);
-    if (DeleteSize && Options.DeleteSizeMismatch) {
+    if (DeleteSize && Options.get(OptionBit::DeleteSizeMismatch)) {
       if (UNLIKELY(DeleteSize != Size))
         reportDeleteSizeMismatch(Ptr, DeleteSize, Size);
     }
 
-    quarantineOrDeallocateChunk(Ptr, &Header, Size);
+    quarantineOrDeallocateChunk(Options, Ptr, &Header, Size);
   }
 
   void *reallocate(void *OldPtr, uptr NewSize, uptr Alignment = MinAlignment) {
     initThreadMaybe();
+    Options Options = Primary.Options.load();
 
     if (UNLIKELY(NewSize >= MaxAllowedMallocSize)) {
-      if (Options.MayReturnNull)
+      if (Options.get(OptionBit::MayReturnNull))
         return nullptr;
       reportAllocationSizeTooBig(NewSize, 0, MaxAllowedMallocSize);
     }
 
     void *OldTaggedPtr = OldPtr;
     OldPtr = untagPointerMaybe(OldPtr);
 
     // The following cases are handled by the C wrappers.
#endif // GWP_ASAN_HOOKS
     Chunk::loadHeader(Cookie, OldPtr, &OldHeader);
 
     if (UNLIKELY(OldHeader.State != Chunk::State::Allocated))
       reportInvalidChunkState(AllocatorAction::Reallocating, OldPtr);
 
     // Pointer has to be allocated with a malloc-type function. Some
     // applications think that it is OK to realloc a memalign'ed pointer, which
     // will trigger this check. It really isn't.
-    if (Options.DeallocTypeMismatch) {
+    if (Options.get(OptionBit::DeallocTypeMismatch)) {
       if (UNLIKELY(OldHeader.OriginOrWasZeroed != Chunk::Origin::Malloc))
         reportDeallocTypeMismatch(AllocatorAction::Reallocating, OldPtr,
                                   OldHeader.OriginOrWasZeroed,
                                   Chunk::Origin::Malloc);
     }
 
     void *BlockBegin = getBlockBegin(OldPtr, &OldHeader);
     uptr BlockEnd;
#endif // GWP_ASAN_HOOKS
     if (reinterpret_cast<uptr>(OldPtr) + NewSize <= BlockEnd) {
       if (NewSize > OldSize || (OldSize - NewSize) < getPageSizeCached()) {
         Chunk::UnpackedHeader NewHeader = OldHeader;
         NewHeader.SizeOrUnusedBytes =
             (ClassId ? NewSize
                      : BlockEnd - (reinterpret_cast<uptr>(OldPtr) + NewSize)) &
             Chunk::SizeOrUnusedBytesMask;
         Chunk::compareExchangeHeader(Cookie, OldPtr, &NewHeader, &OldHeader);
-        if (UNLIKELY(ClassId && useMemoryTagging())) {
+        if (UNLIKELY(ClassId && useMemoryTagging(Options))) {
           resizeTaggedChunk(reinterpret_cast<uptr>(OldTaggedPtr) + OldSize,
                             reinterpret_cast<uptr>(OldTaggedPtr) + NewSize,
                             BlockEnd);
-          storeAllocationStackMaybe(OldPtr);
+          storeAllocationStackMaybe(Options, OldPtr);
         }
         return OldTaggedPtr;
       }
     }
 
     // Otherwise we allocate a new one, and deallocate the old one. Some
     // allocators will allocate an even larger chunk (by a fixed factor) to
     // allow for potential further in-place realloc. The gains of such a trick
     // are currently unclear.
     void *NewPtr = allocate(NewSize, Chunk::Origin::Malloc, Alignment);
     if (NewPtr) {
       const uptr OldSize = getSize(OldPtr, &OldHeader);
       memcpy(NewPtr, OldTaggedPtr, Min(NewSize, OldSize));
-      quarantineOrDeallocateChunk(OldPtr, &OldHeader, OldSize);
+      quarantineOrDeallocateChunk(Options, OldPtr, &OldHeader, OldSize);
     }
     return NewPtr;
   }
 
   // TODO(kostyak): disable() is currently best-effort. There are some small
   //                windows of time when an allocation could still succeed after
   //                this function finishes. We will revisit that later.
   void disable() {
void iterateOverChunks(uptr Base, uptr Size, iterate_callback Callback,
     auto Lambda = [this, From, To, Callback, Arg](uptr Block) {
       if (Block < From || Block >= To)
         return;
       uptr Chunk;
       Chunk::UnpackedHeader Header;
       if (getChunkFromBlock(Block, &Chunk, &Header) &&
           Header.State == Chunk::State::Allocated) {
         uptr TaggedChunk = Chunk;
-        if (useMemoryTagging())
+        if (useMemoryTagging(Primary.Options.load()))
           TaggedChunk = loadTag(Chunk);
         Callback(TaggedChunk, getSize(reinterpret_cast<void *>(Chunk), &Header),
                  Arg);
       }
     };
     Primary.iterateOverBlocks(Lambda);
     Secondary.iterateOverBlocks(Lambda);
 #ifdef GWP_ASAN_HOOKS
     GuardedAlloc.iterate(reinterpret_cast<void *>(Base), Size, Callback, Arg);
 #endif
   }
 
   bool canReturnNull() {
     initThreadMaybe();
-    return Options.MayReturnNull;
+    return Primary.Options.load().get(OptionBit::MayReturnNull);
   }
 
   bool setOption(Option O, sptr Value) {
     initThreadMaybe();
     if (O == Option::MemtagTuning) {
       // Enabling odd/even tags involves a tradeoff between use-after-free
       // detection and buffer overflow detection. Odd/even tags make it more
       // likely for buffer overflows to be detected by increasing the size of
       // the guaranteed "red zone" around the allocation, but on the other hand
       // use-after-free is less likely to be detected because the tag space for
       // any particular chunk is cut in half. Therefore we use this tuning
       // setting to control whether odd/even tags are enabled.
       if (Value == M_MEMTAG_TUNING_BUFFER_OVERFLOW)
-        Options.UseOddEvenTags = true;
+        Primary.Options.set(OptionBit::UseOddEvenTags);
       else if (Value == M_MEMTAG_TUNING_UAF)
-        Options.UseOddEvenTags = false;
+        Primary.Options.clear(OptionBit::UseOddEvenTags);
       return true;
     } else {
       // We leave it to the various sub-components to decide whether or not they
       // want to handle the option, but we do not want to short-circuit
       // execution if one of the setOption was to return false.
       const bool PrimaryResult = Primary.setOption(O, Value);
       const bool SecondaryResult = Secondary.setOption(O, Value);
       const bool RegistryResult = TSDRegistry.setOption(O, Value);
#endif // GWP_ASAN_HOOKS
     if (!Ptr || !isAligned(reinterpret_cast<uptr>(Ptr), MinAlignment))
       return false;
     Ptr = untagPointerMaybe(const_cast<void *>(Ptr));
     Chunk::UnpackedHeader Header;
     return Chunk::isValid(Cookie, Ptr, &Header) &&
            Header.State == Chunk::State::Allocated;
   }
 
-  bool useMemoryTagging() { return Primary.useMemoryTagging(); }
+  bool useMemoryTagging() const {
+    return useMemoryTagging(Primary.Options.load());
+  }
+  static bool useMemoryTagging(Options Options) {
+    return PrimaryT::useMemoryTagging(Options);
+  }
 
   void disableMemoryTagging() { Primary.disableMemoryTagging(); }
 
   void setTrackAllocationStacks(bool Track) {
     initThreadMaybe();
-    Options.TrackAllocationStacks = Track;
+    if (Track)
+      Primary.Options.set(OptionBit::TrackAllocationStacks);
+    else
+      Primary.Options.clear(OptionBit::TrackAllocationStacks);
   }
 
   void setFillContents(FillContentsMode FillContents) {
     initThreadMaybe();
-    Options.FillContents = FillContents;
+    Primary.Options.setFillContentsMode(FillContents);
   }
 
   const char *getStackDepotAddress() const {
     return reinterpret_cast<const char *>(&Depot);
   }
 
   const char *getRegionInfoArrayAddress() const {
     return Primary.getRegionInfoArrayAddress();
private:
   static const sptr MemTagAllocationTidIndex = -1;
   static const sptr MemTagDeallocationTraceIndex = 0;
   static const sptr MemTagDeallocationTidIndex = 1;
   static const sptr MemTagPrevTagIndex = 2;
 
   static const uptr MaxTraceSize = 64;
 
   u32 Cookie;
-
-  struct {
-    u8 MayReturnNull : 1;              // may_return_null
-    FillContentsMode FillContents : 2; // zero_contents, pattern_fill_contents
-    u8 DeallocTypeMismatch : 1;        // dealloc_type_mismatch
-    u8 DeleteSizeMismatch : 1;         // delete_size_mismatch
-    u8 TrackAllocationStacks : 1;
-    u8 UseOddEvenTags : 1;
-    u32 QuarantineMaxChunkSize; // quarantine_max_chunk_size
-  } Options;
+  u32 QuarantineMaxChunkSize;
 
   GlobalStats Stats;
   PrimaryT Primary;
   SecondaryT Secondary;
   QuarantineT Quarantine;
   TSDRegistryT TSDRegistry;
 
 #ifdef GWP_ASAN_HOOKS
#endif // GWP_ASAN_HOOKS
   inline uptr getSize(const void *Ptr, Chunk::UnpackedHeader *Header) {
     const uptr SizeOrUnusedBytes = Header->SizeOrUnusedBytes;
     if (LIKELY(Header->ClassId))
       return SizeOrUnusedBytes;
     return SecondaryT::getBlockEnd(getBlockBegin(Ptr, Header)) -
            reinterpret_cast<uptr>(Ptr) - SizeOrUnusedBytes;
   }
 
-  void quarantineOrDeallocateChunk(void *Ptr, Chunk::UnpackedHeader *Header,
-                                   uptr Size) {
+  void quarantineOrDeallocateChunk(Options Options, void *Ptr,
+                                   Chunk::UnpackedHeader *Header, uptr Size) {
     Chunk::UnpackedHeader NewHeader = *Header;
-    if (UNLIKELY(NewHeader.ClassId && useMemoryTagging())) {
+    if (UNLIKELY(NewHeader.ClassId && useMemoryTagging(Options))) {
       u8 PrevTag = extractTag(loadTag(reinterpret_cast<uptr>(Ptr)));
       if (!TSDRegistry.getDisableMemInit()) {
         uptr TaggedBegin, TaggedEnd;
         const uptr OddEvenMask = computeOddEvenMaskForPointerMaybe(
-            reinterpret_cast<uptr>(getBlockBegin(Ptr, &NewHeader)),
+            Options, reinterpret_cast<uptr>(getBlockBegin(Ptr, &NewHeader)),
             SizeClassMap::getSizeByClassId(NewHeader.ClassId));
         // Exclude the previous tag so that immediate use after free is detected
         // 100% of the time.
         setRandomTag(Ptr, Size, OddEvenMask | (1UL << PrevTag), &TaggedBegin,
                      &TaggedEnd);
       }
       NewHeader.OriginOrWasZeroed = !TSDRegistry.getDisableMemInit();
-      storeDeallocationStackMaybe(Ptr, PrevTag);
+      storeDeallocationStackMaybe(Options, Ptr, PrevTag);
     }
     // If the quarantine is disabled, the actual size of a chunk is 0 or larger
     // than the maximum allowed, we return a chunk directly to the backend.
     // Logical Or can be short-circuited, which introduces unnecessary
     // conditional jumps, so use bitwise Or and let the compiler be clever.
-    const bool BypassQuarantine = !Quarantine.getCacheSize() | !Size |
-                                  (Size > Options.QuarantineMaxChunkSize);
+    const bool BypassQuarantine =
+        !Quarantine.getCacheSize() | !Size | (Size > QuarantineMaxChunkSize);
     if (BypassQuarantine) {
       NewHeader.State = Chunk::State::Available;
       Chunk::compareExchangeHeader(Cookie, Ptr, &NewHeader, Header);
       void *BlockBegin = getBlockBegin(Ptr, &NewHeader);
       const uptr ClassId = NewHeader.ClassId;
       if (LIKELY(ClassId)) {
         bool UnlockRequired;
         auto *TSD = TSDRegistry.getTSDAndLock(&UnlockRequired);
#endif // GWP_ASAN_HOOKS
 
   static uptr getChunkOffsetFromBlock(const char *Block) {
     u32 Offset = 0;
     if (reinterpret_cast<const u32 *>(Block)[0] == BlockMarker)
       Offset = reinterpret_cast<const u32 *>(Block)[1];
     return Offset + Chunk::getHeaderSize();
   }
 
-  void storeAllocationStackMaybe(void *Ptr) {
-    if (!UNLIKELY(Options.TrackAllocationStacks))
+  void storeAllocationStackMaybe(Options Options, void *Ptr) {
+    if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))
       return;
     auto *Ptr32 = reinterpret_cast<u32 *>(Ptr);
     Ptr32[MemTagAllocationTraceIndex] = collectStackTrace();
     Ptr32[MemTagAllocationTidIndex] = getThreadID();
   }
 
-  void storeDeallocationStackMaybe(void *Ptr, uint8_t PrevTag) {
-    if (!UNLIKELY(Options.TrackAllocationStacks))
+  void storeDeallocationStackMaybe(Options Options, void *Ptr,
+                                   uint8_t PrevTag) {
+    if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))
       return;
 
     // Disable tag checks here so that we don't need to worry about zero sized
     // allocations.
     ScopedDisableMemoryTagChecks x;
     auto *Ptr32 = reinterpret_cast<u32 *>(Ptr);
     Ptr32[MemTagDeallocationTraceIndex] = collectStackTrace();
     Ptr32[MemTagDeallocationTidIndex] = getThreadID();

compiler-rt/lib/scudo/standalone/options.h

+#pragma once
+
+#include "atomic_helpers.h"
+#include "common.h"
+
+namespace scudo {
+
+enum class OptionBit {
+  MayReturnNull,
+  FillContents0of2,
+  FillContents1of2,
+  DeallocTypeMismatch,
+  DeleteSizeMismatch,
+  TrackAllocationStacks,
+  UseOddEvenTags,
+  UseMemoryTagging,
+};
+
+struct Options {
+  u32 Val;
+
+  bool get(OptionBit Opt) const { return Val & (1U << static_cast<u32>(Opt)); }
+
+  FillContentsMode getFillContentsMode() const {
+    return static_cast<FillContentsMode>(
+        (Val >> static_cast<u32>(OptionBit::FillContents0of2)) & 3);
+  }
+};
+
+struct AtomicOptions {
+  atomic_u32 Val;
+
+public:
+  Options load() const {
+    return Options{atomic_load(&Val, memory_order_relaxed)};
+  }
+
+  void clear(OptionBit Opt) {
+    atomic_fetch_and(&Val, ~(1U << static_cast<u32>(Opt)),
+                     memory_order_relaxed);
+  }
+
+  void set(OptionBit Opt) {
+    atomic_fetch_or(&Val, 1U << static_cast<u32>(Opt), memory_order_relaxed);
+  }
+
+  void setFillContentsMode(FillContentsMode FillContents) {
+    while (1) {
+      u32 Opts = atomic_load(&Val, memory_order_relaxed);
+      u32 NewOpts = Opts;
+      NewOpts &= ~(3U << static_cast<u32>(OptionBit::FillContents0of2));
+      NewOpts |= static_cast<u32>(FillContents)
+                 << static_cast<u32>(OptionBit::FillContents0of2);
+      if (atomic_compare_exchange_strong(&Val, &Opts, NewOpts,
+                                         memory_order_relaxed))
+        break;
+    }
+  }
+};
+
+} // namespace scudo

compiler-rt/lib/scudo/standalone/primary32.h

 //===-- primary32.h ---------------------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef SCUDO_PRIMARY32_H_
 #define SCUDO_PRIMARY32_H_
 
 #include "bytemap.h"
 #include "common.h"
 #include "list.h"
 #include "local_cache.h"
+#include "options.h"
 #include "release.h"
 #include "report.h"
 #include "stats.h"
 #include "string_utils.h"
 
 namespace scudo {
 
 // SizeClassAllocator32 is an allocator for 32 or 64-bit address space.
uptr releaseToOS() {
     for (uptr I = 0; I < NumClasses; I++) {
       SizeClassInfo *Sci = getSizeClassInfo(I);
       ScopedLock L(Sci->Mutex);
       TotalReleasedBytes += releaseToOSMaybe(Sci, I, /*Force=*/true);
     }
     return TotalReleasedBytes;
   }
 
-  bool useMemoryTagging() { return false; }
+  static bool useMemoryTagging(Options Options) {
+    (void)Options;
+    return false;
+  }
   void disableMemoryTagging() {}
 
   const char *getRegionInfoArrayAddress() const { return nullptr; }
   static uptr getRegionInfoArraySize() { return 0; }
 
   static BlockInfo findNearestBlock(const char *RegionInfoData, uptr Ptr) {
     (void)RegionInfoData;
     (void)Ptr;
     return {};
   }
 
+  AtomicOptions Options;
+
 private:
   static const uptr NumClasses = SizeClassMap::NumClasses;
   static const uptr RegionSize = 1UL << RegionSizeLog;
   static const uptr NumRegions = SCUDO_MMAP_RANGE_SIZE >> RegionSizeLog;
   static const u32 MaxNumBatches = SCUDO_ANDROID ? 4U : 8U;
   typedef FlatByteMap<NumRegions> ByteMap;
 
   struct SizeClassStats {

compiler-rt/lib/scudo/standalone/primary64.h

 //===-- primary64.h ---------------------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef SCUDO_PRIMARY64_H_
 #define SCUDO_PRIMARY64_H_
 
 #include "bytemap.h"
 #include "common.h"
 #include "list.h"
 #include "local_cache.h"
 #include "memtag.h"
+#include "options.h"
 #include "release.h"
 #include "stats.h"
 #include "string_utils.h"
 
 namespace scudo {
 
 // SizeClassAllocator64 is an allocator tuned for 64-bit address space.
 //
for (uptr I = 0; I < NumClasses; I++) {
       // TODO(kostyak): make the lower limit a runtime option
       Region->CanRelease = (I != SizeClassMap::BatchClassId) &&
                            (getSizeByClassId(I) >= (PageSize / 32));
       if (Region->CanRelease)
         Region->ReleaseInfo.LastReleaseAtNs = Time;
     }
     setOption(Option::ReleaseInterval, static_cast<sptr>(ReleaseToOsInterval));
 
-    if (SupportsMemoryTagging)
-      UseMemoryTagging = systemSupportsMemoryTagging();
+    if (SupportsMemoryTagging && systemSupportsMemoryTagging())
+      Options.set(OptionBit::UseMemoryTagging);
   }
   void init(s32 ReleaseToOsInterval) {
     memset(this, 0, sizeof(*this));
     initLinkerInitialized(ReleaseToOsInterval);
   }
 
   void unmapTestOnly() {
     unmap(reinterpret_cast<void *>(PrimaryBase), PrimarySize, UNMAP_ALL, &Data);
uptr releaseToOS() {
     for (uptr I = 0; I < NumClasses; I++) {
       RegionInfo *Region = getRegionInfo(I);
       ScopedLock L(Region->Mutex);
       TotalReleasedBytes += releaseToOSMaybe(Region, I, /*Force=*/true);
     }
     return TotalReleasedBytes;
   }
 
-  bool useMemoryTagging() const {
-    return SupportsMemoryTagging && UseMemoryTagging;
+  static bool useMemoryTagging(Options Options) {
+    return SupportsMemoryTagging && Options.get(OptionBit::UseMemoryTagging);
   }
-  void disableMemoryTagging() { UseMemoryTagging = false; }
+  void disableMemoryTagging() { Options.clear(OptionBit::UseMemoryTagging); }
 
   const char *getRegionInfoArrayAddress() const {
     return reinterpret_cast<const char *>(RegionInfoArray);
   }
 
   static uptr getRegionInfoArraySize() { return sizeof(RegionInfoArray); }
 
   static BlockInfo findNearestBlock(const char *RegionInfoData, uptr Ptr) {
if (MinDistance <= 8192) {
       while (B.BlockBegin < B.RegionBegin)
         B.BlockBegin += B.BlockSize;
       while (B.RegionEnd < B.BlockBegin + B.BlockSize)
         B.BlockBegin -= B.BlockSize;
     }
     return B;
   }
 
+  AtomicOptions Options;
+
 private:
   static const uptr RegionSize = 1UL << RegionSizeLog;
   static const uptr NumClasses = SizeClassMap::NumClasses;
   static const uptr PrimarySize = RegionSize * NumClasses;
 
   // Call map for user memory with at least this size.
   static const uptr MapSizeIncrement = 1UL << 18;
   // Fill at most this number of batches from the newly map'd memory.
struct RegionInfo : UnpaddedRegionInfo {
     char Padding[SCUDO_CACHE_LINE_SIZE -
                  (sizeof(UnpaddedRegionInfo) % SCUDO_CACHE_LINE_SIZE)];
   };
   static_assert(sizeof(RegionInfo) % SCUDO_CACHE_LINE_SIZE == 0, "");
 
   uptr PrimaryBase;
   MapPlatformData Data;
   atomic_s32 ReleaseToOsIntervalMs;
-  bool UseMemoryTagging;
   alignas(SCUDO_CACHE_LINE_SIZE) RegionInfo RegionInfoArray[NumClasses];
 
   RegionInfo *getRegionInfo(uptr ClassId) {
     DCHECK_LT(ClassId, NumClasses);
     return &RegionInfoArray[ClassId];
   }
 
   uptr getRegionBaseByClassId(uptr ClassId) const {
if (TotalUserBytes > MappedUser) {
         }
         return nullptr;
       }
       if (UNLIKELY(MappedUser == 0))
         Region->Data = Data;
       if (UNLIKELY(!map(reinterpret_cast<void *>(RegionBeg + MappedUser),
                         UserMapSize, "scudo:primary",
                         MAP_ALLOWNOMEM | MAP_RESIZABLE |
-                            (useMemoryTagging() ? MAP_MEMTAG : 0),
+                            (useMemoryTagging(Options.load()) ? MAP_MEMTAG : 0),
                         &Region->Data)))
         return nullptr;
       Region->MappedUser += UserMapSize;
       C->getStats().add(StatMapped, UserMapSize);
     }
 
     const u32 NumberOfBlocks = Min(
         MaxNumBatches * MaxCount,

compiler-rt/lib/scudo/standalone/wrappers_c.inc

 //===-- wrappers_c.inc ------------------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef SCUDO_PREFIX
 #error "Define SCUDO_PREFIX prior to including this file!"
 #endif
 
 // malloc-type functions have to be aligned to std::max_align_t. This is
 // distinct from (1U << SCUDO_MIN_ALIGNMENT_LOG), since C++ new-type functions
 // do not have to abide by the same requirement.
 #ifndef SCUDO_MALLOC_ALIGNMENT
 #define SCUDO_MALLOC_ALIGNMENT FIRST_32_SECOND_64(8U, 16U)
 #endif
 
 extern "C" {
 
 INTERFACE WEAK void *SCUDO_PREFIX(calloc)(size_t nmemb, size_t size) {
   scudo::uptr Product;
   if (UNLIKELY(scudo::checkForCallocOverflow(size, nmemb, &Product))) {
     if (SCUDO_ALLOCATOR.canReturnNull()) {
       errno = ENOMEM;
       return nullptr;
     }
     scudo::reportCallocOverflow(nmemb, size);
   }
   return scudo::setErrnoOnNull(SCUDO_ALLOCATOR.allocate(
       Product, scudo::Chunk::Origin::Malloc, SCUDO_MALLOC_ALIGNMENT, true));
 }
 
 INTERFACE WEAK void SCUDO_PREFIX(free)(void *ptr) {
   SCUDO_ALLOCATOR.deallocate(ptr, scudo::Chunk::Origin::Malloc);
 }
 
 INTERFACE WEAK struct SCUDO_MALLINFO SCUDO_PREFIX(mallinfo)(void) {
if (sizes[i])
       fprintf(stream, "<alloc size=\"%lu\" count=\"%lu\"/>\n", i, sizes[i]);
   fputs("</malloc>\n", stream);
   SCUDO_PREFIX(free)(sizes);
   return 0;
 }
 
 // Disable memory tagging for the heap. The caller must disable memory tag
 // checks globally (e.g. by clearing TCF0 on aarch64) before calling this
-// function, and may not re-enable them after calling the function. The program
-// must be single threaded at the point when the function is called.
+// function, and may not re-enable them after calling the function.
 INTERFACE WEAK void SCUDO_PREFIX(malloc_disable_memory_tagging)() {
   SCUDO_ALLOCATOR.disableMemoryTagging();
 }
 
 // Sets whether scudo records stack traces and other metadata for allocations
 // and deallocations. This function only has an effect if the allocator and
-// hardware support memory tagging. The program must be single threaded at the
-// point when the function is called.
+// hardware support memory tagging.
 INTERFACE WEAK void
 SCUDO_PREFIX(malloc_set_track_allocation_stacks)(int track) {
   SCUDO_ALLOCATOR.setTrackAllocationStacks(track);
 }
 
-// Sets whether scudo zero-initializes all allocated memory. The program must
-// be single threaded at the point when the function is called.
+// Sets whether scudo zero-initializes all allocated memory.
 INTERFACE WEAK void SCUDO_PREFIX(malloc_set_zero_contents)(int zero_contents) {
   SCUDO_ALLOCATOR.setFillContents(zero_contents ? scudo::ZeroFill
                                                 : scudo::NoFill);
 }
 
-// Sets whether scudo pattern-initializes all allocated memory. The program must
-// be single threaded at the point when the function is called.
+// Sets whether scudo pattern-initializes all allocated memory.
 INTERFACE WEAK void
 SCUDO_PREFIX(malloc_set_pattern_fill_contents)(int pattern_fill_contents) {
   SCUDO_ALLOCATOR.setFillContents(
       pattern_fill_contents ? scudo::PatternOrZeroFill : scudo::NoFill);
 }
 
 } // extern "C"