scudo: Add an API for disabling memory initialization per-thread.

[pcc]

Summary

Here "memory initialization" refers to zero- or pattern-init on
non-MTE hardware, or (where possible to avoid) memory tagging on MTE
hardware. With shared TSD the per-thread memory initialization state
is stored in bit 0 of the TLS slot, similar to PointerIntPair in LLVM.

[pcc]

This is the patch that I've been using to stress test my implementation:

diff --git a/compiler-rt/lib/scudo/standalone/combined.h b/compiler-rt/lib/scudo/standalone/combined.h
index 6c39b8f361e7..54ea5f487ac9 100644
--- a/compiler-rt/lib/scudo/standalone/combined.h
+++ b/compiler-rt/lib/scudo/standalone/combined.h
@@ -9,6 +9,8 @@
 #ifndef SCUDO_COMBINED_H_
 #define SCUDO_COMBINED_H_
 
+#include <stdlib.h>
+
 #include "chunk.h"
 #include "common.h"
 #include "flags.h"
@@ -268,6 +270,14 @@ public:
                           bool ZeroContents = false) {
     initThreadMaybe();
 
+    {
+      int R = rand();
+      if (R % 16 == 0)
+        TSDRegistry.setOption(Option::ThreadDisableMemInit, 0);
+      if (R % 16 == 1)
+        TSDRegistry.setOption(Option::ThreadDisableMemInit, 1);
+    }
+
 #ifdef GWP_ASAN_HOOKS
     if (UNLIKELY(GuardedAlloc.shouldSample())) {
       if (void *Ptr = GuardedAlloc.allocate(roundUpTo(Size, Alignment)))

With this applied and booting Android on FVP I sometimes get an application crash in com.android.settings or com.android.systemui. The tombstone starts like this:

Build fingerprint: 'Android/fvp/fvpbase:S/AOSP.MASTER/eng.pcc.20200813.203906:eng/test-keys'
Revision: '0'
ABI: 'arm64'
Timestamp: 2020-09-15 23:35:00.120584420+0000
pid: 1366, tid: 1707, name: RenderThread  >>> com.android.systemui <<<
uid: 10100
tagged_addr_ctrl: 000000000007fff5
signal 11 (SIGSEGV), code 1 (SEGV_MAPERR), fault addr 0x35338100000010
    x0  0000007a599b10b0  x1  0a00007b9dcb2210  x2  0000000000000003  x3  0000000000000003
    x4  0000000000000000  x5  0000007a599b1400  x6  00000000000003ff  x7  0000000000000400
    x8  0000000000000002  x9  0000000000000003  x10 0000000000000002  x11 0000000000000013
    x12 0600007afdd915a0  x13 0000000000000002  x14 0035338100000000  x15 0000000000000007
    x16 0000007d6f9db4a0  x17 0000007d6f76c7c4  x18 0000007a50878000  x19 0a00007b9dcb2210
    x20 0000007a599b10b0  x21 0000007a599b0e40  x22 0000000000000003  x23 0000000000000003
    x24 0000007a599b0d78  x25 0000007a599b3000  x26 0000000000000000  x27 0d00007b9dcc7578
    x28 0000000000000000  x29 0000007a599b0cf0
    lr  0000007d6f814f38  sp  0000007a599b0ce0  pc  0000007d6f7e3e04  pst 0000000020001000

backtrace:
      #00 pc 00000000003e1e04  /system/lib64/libhwui.so (GrResourceAllocator::addInterval(GrSurfaceProxy*, unsigned int, unsigned int)+132) (BuildId: ecfd2fad5f2e390ec6cf7b8577e85b87)
      #01 pc 0000000000412f34  /system/lib64/libhwui.so (GrProcessorSet::visitProxies(std::__1::function<void (GrSurfaceProxy*)> const&) const+304) (BuildId: ecfd2fad5f2e390ec6cf7b8577e85b87)
      #02 pc 00000000003ded90  /system/lib64/libhwui.so (GrRenderTargetOpList::OpChain::visitProxies(std::__1::function<void (GrSurfaceProxy*)> const&, GrOp::VisitorType) const+76) (BuildId: ecfd2fad5f2e390ec6cf7b8577e85b87)
      #03 pc 00000000003e0c60  /system/lib64/libhwui.so (GrRenderTargetOpList::gatherProxyIntervals(GrResourceAllocator*) const+296) (BuildId: ecfd2fad5f2e390ec6cf7b8577e85b87)
      #04 pc 00000000003b9988  /system/lib64/libhwui.so (GrDrawingManager::flush(GrSurfaceProxy*, SkSurface::BackendSurfaceAccess, GrFlushFlags, int, GrBackendSemaphore*, void (*)(void*), void*)+1980) (BuildId: ecfd2fad5f2e390ec6cf7b8577e85b87)
      #05 pc 00000000003ba2b8  /system/lib64/libhwui.so (GrDrawingManager::prepareSurfaceForExternalIO(GrSurfaceProxy*, SkSurface::BackendSurfaceAccess, GrFlushFlags, int, GrBackendSemaphore*, void (*)(void*), void*)+244) (BuildId: ecfd2fad5f2e390ec6cf7b8577e85b87)
      #06 pc 00000000003dd038  /system/lib64/libhwui.so (GrRenderTargetContext::prepareForExternalIO(SkSurface::BackendSurfaceAccess, GrFlushFlags, int, GrBackendSemaphore*, void (*)(void*), void*)+252) (BuildId: ecfd2fad5f2e390ec6cf7b8577e85b87)
      #07 pc 000000000018b0f4  /system/lib64/libhwui.so (android::uirenderer::skiapipeline::SkiaPipeline::renderFrame(android::uirenderer::LayerUpdateQueue const&, SkRect const&, std::__1::vector<android::sp<android::uirenderer::RenderNode>, std::__1::allocator<android::sp<android::uirenderer::RenderNode> > > const&, bool, android::uirenderer::Rect const&, sk_sp<SkSurface>, SkMatrix const&)+208) (BuildId: ecfd2fad5f2e390ec6cf7b8577e85b87)
      #08 pc 0000000000188d04  /system/lib64/libhwui.so (android::uirenderer::skiapipeline::SkiaOpenGLPipeline::draw(android::uirenderer::renderthread::Frame const&, SkRect const&, SkRect const&, android::uirenderer::LightGeometry const&, android::uirenderer::LayerUpdateQueue*, android::uirenderer::Rect const&, bool, android::uirenderer::LightInfo const&, std::__1::vector<android::sp<android::uirenderer::RenderNode>, std::__1::allocator<android::sp<android::uirenderer::RenderNode> > > const&, android::uirenderer::FrameInfoVisualizer*)+496) (BuildId: ecfd2fad5f2e390ec6cf7b8577e85b87)
      #09 pc 00000000001943e4  /system/lib64/libhwui.so (android::uirenderer::renderthread::CanvasContext::draw()+392) (BuildId: ecfd2fad5f2e390ec6cf7b8577e85b87)
      #10 pc 000000000019636c  /system/lib64/libhwui.so (android::uirenderer::renderthread::DrawFrameTask::run()+428) (BuildId: ecfd2fad5f2e390ec6cf7b8577e85b87)
      #11 pc 0000000000179dfc  /system/lib64/libhwui.so (android::uirenderer::WorkQueue::process()+400) (BuildId: ecfd2fad5f2e390ec6cf7b8577e85b87)
      #12 pc 00000000001a56a8  /system/lib64/libhwui.so (android::uirenderer::renderthread::RenderThread::threadLoop()+128) (BuildId: ecfd2fad5f2e390ec6cf7b8577e85b87)
      #13 pc 0000000000014444  /system/lib64/libutils.so (android::Thread::_threadLoop(void*)+260) (BuildId: dfc95e0a7c124e5bf890590bb0f3c55f)
      #14 pc 0000000000013cf8  /system/lib64/libutils.so (thread_data_t::trampoline(void*)+424) (BuildId: dfc95e0a7c124e5bf890590bb0f3c55f)
      #15 pc 00000000000af5fc  /apex/com.android.runtime/lib64/bionic/libc.so (__pthread_start(void*)+344) (BuildId: 6cd1bc2d17bd5e5999e53add462790e2)
      #16 pc 000000000004f448  /apex/com.android.runtime/lib64/bionic/libc.so (__start_thread+64) (BuildId: 6cd1bc2d17bd5e5999e53add462790e2)

And the top frame symbolizes to this line of code:
https://cs.android.com/android/platform/superproject/+/master:external/skia/src/core/SkTDynamicHash.h;l=98
which is accessing a pointer from a calloc'd region and expecting it to either be zero or a stored value, indicating that the memory was not zero initialized correctly.

[hctim]

compiler-rt/lib/scudo/standalone/combined.h

498

Isn't this now broken under dealloc_type_mismatch and MTE?

556

same here, broken under dealloc_type_mismatch when memory is zeroed?

[pcc]

compiler-rt/lib/scudo/standalone/combined.h

498

No, because the field is intended to only have the "was zeroed" meaning while the chunk is not allocated. Note that the field is read before the call to quarantineOrDeallocateChunk later in this function causes it to take the "was zeroed" meaning.

556

This is before the call to quarantineOrDeallocateChunk.

[hctim]

compiler-rt/lib/scudo/standalone/combined.h

498

I see - thanks for the clarification. Maybe OriginIfAliveOrWasZeroedIfDead? A bit wordy (can't think of anything more concise right now), but otherwise it's a little hard to infer this which meaning applies at which time (especially given that SizeOrUnusedBytes has different semantics). Either that or a comment in the declaration explaining the semantics would probably suffice.

[eugenis]

compiler-rt/lib/scudo/standalone/combined.h

417

resizeTaggedChunk does not zero memory. I think this memset needs to cover the entire allocation?

417

Ah no, I misread. Ignore this.

428

Rename setRandomTag to setRandomTagAndZeroData ?

[pcc]

compiler-rt/lib/scudo/standalone/combined.h

417

I think the problem is related to this line actually. resizeTaggedChunk rounds up to 16 before it starts zeroing, so the memset needs to stop at the rounded-up original bound, otherwise it can leave the end of the original last granule uninitialized. I am testing this diff:

@@ -414,12 +424,18 @@ public:
             // UAF tag. But if tagging was disabled per-thread when the memory
             // was freed, it would not have been retagged and thus zeroed, and
             // therefore it needs to be zeroed now.
-            memset(TaggedPtr, 0, Min(Size, PrevEnd - TaggedUserPtr));
+            memset(TaggedPtr, 0,
+                   Min(Size, roundUpTo(PrevEnd - TaggedUserPtr,
+                                       archMemoryTagGranuleSize())));
           } else if (Size) {
             // Clear any stack metadata that may have previously been stored in
             // the chunk data.

428

Does that really need to be in the name of the function or would a comment be enough? I think most of the functions in memtag.h end up zeroing parts of the memory that they touch.

[pcc]

Fix bug, add test, address review comment

[pcc]

compiler-rt/lib/scudo/standalone/combined.h

417

Okay, this looks like this fixed the problem (stress tested on Android/FVP with a more rigorous version of my stress testing patch that verifies that calloc allocations are zeroed at allocation time). Also added a test case that fails with the previous version of my patch.

498

Added a comment in the declaration.

[eugenis]

compiler-rt/lib/scudo/standalone/combined.h

410

This implies that MTE mode does not support pattern init - and it's also clear from the rest of the code. Should we have setFillContents do something about that? It does not have a return value, but it could complain in stderr (not sure it's a good idea) or fall back to zero-init.

428

Thanks. Does not have to be in the function name - it's just was something that made me go and look at the implementation.

[pcc]

compiler-rt/lib/scudo/standalone/combined.h

410

I think it would be fine to allow setting it to PatternOrZeroFill to go through without warning. As the name implies, it would be valid to implement it using zero init.

[eugenis]

LGTM

compiler-rt/lib/scudo/standalone/combined.h

410

Yes, good point.

compiler-rt/lib/scudo/standalone/tests/combined_test.cpp

535

make it a named constant, or use Ptrs.size()

[cryptoad]

compiler-rt/lib/scudo/standalone/combined.h

1034

Looks like some tabs snuck in maybe? You might want to clang-format -style=llvm

[eugenis]

compiler-rt/lib/scudo/standalone/combined.h

1034

No, it's a new thing that Phabricator is doing when the code is re-indented :)

[cryptoad]

compiler-rt/lib/scudo/standalone/combined.h

1034

Ack!

compiler-rt/lib/scudo/standalone/chunk.h

 
 enum State : u8 { Available = 0, Allocated = 1, Quarantined = 2 };
 
 typedef u64 PackedHeader;
 // Update the 'Mask' constants to reflect changes in this structure.
 struct UnpackedHeader {
   uptr ClassId : 8;
   u8 State : 2;
-  u8 Origin : 2;
+  // Origin if State == Allocated, or WasZeroed otherwise.
+  u8 OriginOrWasZeroed : 2;
   uptr SizeOrUnusedBytes : 20;
   uptr Offset : 16;
   uptr Checksum : 16;
 };
 typedef atomic_u64 AtomicPackedHeader;
 static_assert(sizeof(UnpackedHeader) == sizeof(PackedHeader), "");
 
 // Those constants are required to silence some -Werror=conversion errors when

compiler-rt/lib/scudo/standalone/combined.h

 
 #ifdef GWP_ASAN_HOOKS
     if (UNLIKELY(GuardedAlloc.shouldSample())) {
       if (void *Ptr = GuardedAlloc.allocate(roundUpTo(Size, Alignment)))
         return Ptr;
     }
 #endif // GWP_ASAN_HOOKS
 
-    const FillContentsMode FillContents =
-        ZeroContents ? ZeroFill : Options.FillContents;
+    const FillContentsMode FillContents = ZeroContents ? ZeroFill
+                                          : TSDRegistry.getDisableMemInit()
+                                              ? NoFill
+                                              : Options.FillContents;
 
     if (UNLIKELY(Alignment > MaxAlignment)) {
       if (Options.MayReturnNull)
         return nullptr;
       reportAlignmentTooBig(Alignment, MaxAlignment);
     }
     if (UNLIKELY(Alignment < MinAlignment))
       Alignment = MinAlignment;
if (LIKELY(ClassId)) {
             PrevUserPtr == UserPtr &&
             (TaggedUserPtr = loadTag(UserPtr)) != UserPtr) {
           uptr PrevEnd = TaggedUserPtr + Header.SizeOrUnusedBytes;
           const uptr NextPage = roundUpTo(TaggedUserPtr, getPageSizeCached());
           if (NextPage < PrevEnd && loadTag(NextPage) != NextPage)
             PrevEnd = NextPage;
           TaggedPtr = reinterpret_cast<void *>(TaggedUserPtr);
           resizeTaggedChunk(PrevEnd, TaggedUserPtr + Size, BlockEnd);
-          if (Size) {
+          if (UNLIKELY(FillContents != NoFill && !Header.OriginOrWasZeroed)) {
+            // If an allocation needs to be zeroed (i.e. calloc) we can normally
+            // avoid zeroing the memory now since we can rely on memory having
+            // been zeroed on free, as this is normally done while setting the
+            // UAF tag. But if tagging was disabled per-thread when the memory
+            // was freed, it would not have been retagged and thus zeroed, and
+            // therefore it needs to be zeroed now.
+            memset(TaggedPtr, 0,
+                   Min(Size, roundUpTo(PrevEnd - TaggedUserPtr,
+                                       archMemoryTagGranuleSize())));
+          } else if (Size) {
             // Clear any stack metadata that may have previously been stored in
             // the chunk data.
             memset(TaggedPtr, 0, archMemoryTagGranuleSize());
           }
         } else {
           const uptr OddEvenMask =
               computeOddEvenMaskForPointerMaybe(BlockUptr, BlockSize);
           TaggedPtr = prepareTaggedChunk(Ptr, Size, OddEvenMask, BlockEnd);
         }
         storeAllocationStackMaybe(Ptr);
       } else if (UNLIKELY(FillContents != NoFill)) {
         // This condition is not necessarily unlikely, but since memset is
         // costly, we might as well mark it as such.
         memset(Block, FillContents == ZeroFill ? 0 : PatternFillByte,
                PrimaryT::getSizeByClassId(ClassId));
       }
     }
 
     Chunk::UnpackedHeader Header = {};
     if (UNLIKELY(UnalignedUserPtr != UserPtr)) {
       const uptr Offset = UserPtr - UnalignedUserPtr;
       DCHECK_GE(Offset, 2 * sizeof(u32));
       // The BlockMarker has no security purpose, but is specifically meant for
       // the chunk iteration function that can be used in debugging situations.
       // It is the only situation where we have to locate the start of a chunk
       // based on its block address.
       reinterpret_cast<u32 *>(Block)[0] = BlockMarker;
       reinterpret_cast<u32 *>(Block)[1] = static_cast<u32>(Offset);
       Header.Offset = (Offset >> MinAlignmentLog) & Chunk::OffsetMask;
     }
     Header.ClassId = ClassId & Chunk::ClassIdMask;
     Header.State = Chunk::State::Allocated;
-    Header.Origin = Origin & Chunk::OriginMask;
+    Header.OriginOrWasZeroed = Origin & Chunk::OriginMask;
     Header.SizeOrUnusedBytes =
         (ClassId ? Size : SecondaryBlockEnd - (UserPtr + Size)) &
         Chunk::SizeOrUnusedBytesMask;
     Chunk::storeHeader(Cookie, Ptr, &Header);
 
     if (&__scudo_allocate_hook)
       __scudo_allocate_hook(TaggedPtr, Size);
 
#endif // GWP_ASAN_HOOKS
     Ptr = untagPointerMaybe(Ptr);
 
     Chunk::UnpackedHeader Header;
     Chunk::loadHeader(Cookie, Ptr, &Header);
 
     if (UNLIKELY(Header.State != Chunk::State::Allocated))
       reportInvalidChunkState(AllocatorAction::Deallocating, Ptr);
     if (Options.DeallocTypeMismatch) {
-      if (Header.Origin != Origin) {
+      if (Header.OriginOrWasZeroed != Origin) {
         // With the exception of memalign'd chunks, that can be still be free'd.
-        if (UNLIKELY(Header.Origin != Chunk::Origin::Memalign ||
+        if (UNLIKELY(Header.OriginOrWasZeroed != Chunk::Origin::Memalign ||
                      Origin != Chunk::Origin::Malloc))
           reportDeallocTypeMismatch(AllocatorAction::Deallocating, Ptr,
-                                    Header.Origin, Origin);
+                                    Header.OriginOrWasZeroed, Origin);
       }
     }
 
     const uptr Size = getSize(Ptr, &Header);
     if (DeleteSize && Options.DeleteSizeMismatch) {
       if (UNLIKELY(DeleteSize != Size))
         reportDeleteSizeMismatch(Ptr, DeleteSize, Size);
     }
#endif // GWP_ASAN_HOOKS
 
     if (UNLIKELY(OldHeader.State != Chunk::State::Allocated))
       reportInvalidChunkState(AllocatorAction::Reallocating, OldPtr);
 
     // Pointer has to be allocated with a malloc-type function. Some
     // applications think that it is OK to realloc a memalign'ed pointer, which
     // will trigger this check. It really isn't.
     if (Options.DeallocTypeMismatch) {
-      if (UNLIKELY(OldHeader.Origin != Chunk::Origin::Malloc))
+      if (UNLIKELY(OldHeader.OriginOrWasZeroed != Chunk::Origin::Malloc))
         reportDeallocTypeMismatch(AllocatorAction::Reallocating, OldPtr,
-                                  OldHeader.Origin, Chunk::Origin::Malloc);
+                                  OldHeader.OriginOrWasZeroed,
+                                  Chunk::Origin::Malloc);
     }
 
     void *BlockBegin = getBlockBegin(OldPtr, &OldHeader);
     uptr BlockEnd;
     uptr OldSize;
     const uptr ClassId = OldHeader.ClassId;
     if (LIKELY(ClassId)) {
       BlockEnd = reinterpret_cast<uptr>(BlockBegin) +
return SecondaryT::getBlockEnd(getBlockBegin(Ptr, Header)) -
            reinterpret_cast<uptr>(Ptr) - SizeOrUnusedBytes;
   }
 
   void quarantineOrDeallocateChunk(void *Ptr, Chunk::UnpackedHeader *Header,
                                    uptr Size) {
     Chunk::UnpackedHeader NewHeader = *Header;
     if (UNLIKELY(NewHeader.ClassId && useMemoryTagging())) {
       u8 PrevTag = extractTag(loadTag(reinterpret_cast<uptr>(Ptr)));
+      if (!TSDRegistry.getDisableMemInit()) {
         uptr TaggedBegin, TaggedEnd;
         const uptr OddEvenMask = computeOddEvenMaskForPointerMaybe(
             reinterpret_cast<uptr>(getBlockBegin(Ptr, &NewHeader)),
             SizeClassMap::getSizeByClassId(NewHeader.ClassId));
         // Exclude the previous tag so that immediate use after free is detected
         // 100% of the time.
         setRandomTag(Ptr, Size, OddEvenMask | (1UL << PrevTag), &TaggedBegin,
                      &TaggedEnd);
+      }
+      NewHeader.OriginOrWasZeroed = !TSDRegistry.getDisableMemInit();
       storeDeallocationStackMaybe(Ptr, PrevTag);
     }
     // If the quarantine is disabled, the actual size of a chunk is 0 or larger
     // than the maximum allowed, we return a chunk directly to the backend.
     // Logical Or can be short-circuited, which introduces unnecessary
     // conditional jumps, so use bitwise Or and let the compiler be clever.
     const bool BypassQuarantine = !Quarantine.getCacheSize() | !Size |
                                   (Size > Options.QuarantineMaxChunkSize);

compiler-rt/lib/scudo/standalone/common.h

struct BlockInfo {
   uptr BlockSize;
   uptr RegionBegin;
   uptr RegionEnd;
 };
 
 enum class Option : u8 {
   ReleaseInterval,      // Release to OS interval in milliseconds.
   MemtagTuning,         // Whether to tune tagging for UAF or overflow.
+  ThreadDisableMemInit, // Whether to disable automatic heap initialization and,
+                        // where possible, memory tagging, on this thread.
   MaxCacheEntriesCount, // Maximum number of blocks that can be cached.
   MaxCacheEntrySize,    // Maximum size of a block that can be cached.
   MaxTSDsCount,         // Number of usable TSDs for the shared registry.
 };
 
 constexpr unsigned char PatternFillByte = 0xAB;
 
 enum FillContentsMode {

compiler-rt/lib/scudo/standalone/include/scudo/interface.h

 
 // Tune the allocator's choice of memory tags to make it more likely that
 // a certain class of memory errors will be detected. The value argument should
 // be one of the enumerators of the scudo_memtag_tuning enum below.
 #ifndef M_MEMTAG_TUNING
 #define M_MEMTAG_TUNING -102
 #endif
 
+// Per-thread memory initialization tuning. The value argument should be one of:
+// 1: Disable automatic heap initialization and, where possible, memory tagging,
+//    on this thread.
+// 0: Normal behavior.
+#ifndef M_THREAD_DISABLE_MEM_INIT
+#define M_THREAD_DISABLE_MEM_INIT -103
+#endif
+
 #ifndef M_CACHE_COUNT_MAX
 #define M_CACHE_COUNT_MAX -200
 #endif
 
 #ifndef M_CACHE_SIZE_MAX
 #define M_CACHE_SIZE_MAX -201
 #endif
 

compiler-rt/lib/scudo/standalone/tests/chunk_test.cpp

TEST(ScudoChunkTest, ChunkBasic) {
   EXPECT_DEATH(scudo::Chunk::loadHeader(InvalidCookie, P, &Header), "");
   free(Block);
 }
 
 TEST(ScudoChunkTest, ChunkCmpXchg) {
   initChecksum();
   const scudo::uptr Size = 0x100U;
   scudo::Chunk::UnpackedHeader OldHeader = {};
-  OldHeader.Origin = scudo::Chunk::Origin::Malloc;
+  OldHeader.OriginOrWasZeroed = scudo::Chunk::Origin::Malloc;
   OldHeader.ClassId = 0x42U;
   OldHeader.SizeOrUnusedBytes = Size;
   OldHeader.State = scudo::Chunk::State::Allocated;
   void *Block = malloc(HeaderSize + Size);
   void *P = reinterpret_cast<void *>(reinterpret_cast<scudo::uptr>(Block) +
                                      HeaderSize);
   scudo::Chunk::storeHeader(Cookie, P, &OldHeader);
   memset(P, 'A', Size);

compiler-rt/lib/scudo/standalone/tests/combined_test.cpp

for (unsigned I = 0; I != 65536; ++I) {
         CheckOddEven(P, P + Size);
         break;
       }
       Ptrs.insert(P);
     }
     EXPECT_TRUE(Found);
   }
 }
+
+TEST(ScudoCombinedTest, DisableMemInit) {
+  using AllocatorT = TestAllocator<scudo::AndroidConfig>;
+  using SizeClassMap = AllocatorT::PrimaryT::SizeClassMap;
+  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());
+
+  std::vector<void *> Ptrs(65536, nullptr);
+
+  Allocator->setOption(scudo::Option::ThreadDisableMemInit, 1);
+
+  constexpr scudo::uptr MinAlignLog = FIRST_32_SECOND_64(3U, 4U);
+
+  // Test that if mem-init is disabled on a thread, calloc should still work as
+  // expected. This is tricky to ensure when MTE is enabled, so this test tries
+  // to exercise the relevant code on our MTE path.
+  for (scudo::uptr ClassId = 1U; ClassId <= 8; ClassId++) {
+    const scudo::uptr Size =
+        SizeClassMap::getSizeByClassId(ClassId) - scudo::Chunk::getHeaderSize();
+    if (Size < 8)
+      continue;
+    for (unsigned I = 0; I != 65536; ++I) {
+      Ptrs[I] = Allocator->allocate(Size, Origin);
+      memset(Ptrs[I], 0xaa, Size);
+    }
+    for (unsigned I = 0; I != 65536; ++I)
+      Allocator->deallocate(Ptrs[I], Origin, Size);
+    for (unsigned I = 0; I != 65536; ++I) {
+      Ptrs[I] = Allocator->allocate(Size - 8, Origin);
+      memset(Ptrs[I], 0xbb, Size - 8);
+    }
+    for (unsigned I = 0; I != 65536; ++I)
+      Allocator->deallocate(Ptrs[I], Origin, Size - 8);
+    for (unsigned I = 0; I != 65536; ++I) {
+      Ptrs[I] = Allocator->allocate(Size, Origin, 1U << MinAlignLog, true);
+      for (scudo::uptr J = 0; J < Size; ++J)
+        ASSERT_EQ((reinterpret_cast<char *>(Ptrs[I]))[J], 0);
+    }
+  }
+
+  Allocator->setOption(scudo::Option::ThreadDisableMemInit, 0);
+}

compiler-rt/lib/scudo/standalone/tsd_exclusive.h

 //===-- tsd_exclusive.h -----------------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef SCUDO_TSD_EXCLUSIVE_H_
 #define SCUDO_TSD_EXCLUSIVE_H_
 
 #include "tsd.h"
 
 namespace scudo {
 
-enum class ThreadState : u8 {
+struct ThreadState {
+  bool DisableMemInit : 1;
+  enum {
     NotInitialized = 0,
     Initialized,
     TornDown,
+  } InitState : 2;
 };
 
 template <class Allocator> void teardownThread(void *Ptr);
 
 template <class Allocator> struct TSDRegistryExT {
   void initLinkerInitialized(Allocator *Instance) {
     Instance->initLinkerInitialized();
     CHECK_EQ(pthread_key_create(&PThreadKey, teardownThread<Allocator>), 0);
     FallbackTSD.initLinkerInitialized(Instance);
     Initialized = true;
   }
   void init(Allocator *Instance) {
     memset(this, 0, sizeof(*this));
     initLinkerInitialized(Instance);
   }
 
   void unmapTestOnly() {}
 
   ALWAYS_INLINE void initThreadMaybe(Allocator *Instance, bool MinimalInit) {
-    if (LIKELY(State != ThreadState::NotInitialized))
+    if (LIKELY(State.InitState != ThreadState::NotInitialized))
       return;
     initThread(Instance, MinimalInit);
   }
 
   ALWAYS_INLINE TSD<Allocator> *getTSDAndLock(bool *UnlockRequired) {
-    if (LIKELY(State == ThreadState::Initialized &&
+    if (LIKELY(State.InitState == ThreadState::Initialized &&
                !atomic_load(&Disabled, memory_order_acquire))) {
       *UnlockRequired = false;
       return &ThreadTSD;
     }
     FallbackTSD.lock();
     *UnlockRequired = true;
     return &FallbackTSD;
   }
 
   // To disable the exclusive TSD registry, we effectively lock the fallback TSD
   // and force all threads to attempt to use it instead of their local one.
   void disable() {
     Mutex.lock();
     FallbackTSD.lock();
     atomic_store(&Disabled, 1U, memory_order_release);
   }
 
   void enable() {
     atomic_store(&Disabled, 0U, memory_order_release);
     FallbackTSD.unlock();
     Mutex.unlock();
   }
 
   bool setOption(Option O, UNUSED sptr Value) {
+    if (O == Option::ThreadDisableMemInit)
+      State.DisableMemInit = Value;
     if (O == Option::MaxTSDsCount)
       return false;
     return true;
   }
 
+  bool getDisableMemInit() { return State.DisableMemInit; }
+
 private:
   void initOnceMaybe(Allocator *Instance) {
     ScopedLock L(Mutex);
     if (LIKELY(Initialized))
       return;
     initLinkerInitialized(Instance); // Sets Initialized.
   }
 
   // Using minimal initialization allows for global initialization while keeping
   // the thread specific structure untouched. The fallback structure will be
   // used instead.
   NOINLINE void initThread(Allocator *Instance, bool MinimalInit) {
     initOnceMaybe(Instance);
     if (UNLIKELY(MinimalInit))
       return;
     CHECK_EQ(
         pthread_setspecific(PThreadKey, reinterpret_cast<void *>(Instance)), 0);
     ThreadTSD.initLinkerInitialized(Instance);
-    State = ThreadState::Initialized;
+    State.InitState = ThreadState::Initialized;
     Instance->callPostInitCallback();
   }
 
   pthread_key_t PThreadKey;
   bool Initialized;
   atomic_u8 Disabled;
   TSD<Allocator> FallbackTSD;
   HybridMutex Mutex;
template <class Allocator> void teardownThread(void *Ptr) {
   if (TSDRegistryT::ThreadTSD.DestructorIterations > 1) {
     TSDRegistryT::ThreadTSD.DestructorIterations--;
     // If pthread_setspecific fails, we will go ahead with the teardown.
     if (LIKELY(pthread_setspecific(Instance->getTSDRegistry()->PThreadKey,
                                    Ptr) == 0))
       return;
   }
   TSDRegistryT::ThreadTSD.commitBack(Instance);
-  TSDRegistryT::State = ThreadState::TornDown;
+  TSDRegistryT::State.InitState = ThreadState::TornDown;
 }
 
 } // namespace scudo
 
 #endif // SCUDO_TSD_EXCLUSIVE_H_

compiler-rt/lib/scudo/standalone/tsd_shared.h

void enable() {
     for (s32 I = static_cast<s32>(TSDsArraySize - 1); I >= 0; I--)
       TSDs[I].unlock();
     Mutex.unlock();
   }
 
   bool setOption(Option O, sptr Value) {
     if (O == Option::MaxTSDsCount)
       return setNumberOfTSDs(static_cast<u32>(Value));
+    if (O == Option::ThreadDisableMemInit)
+      setDisableMemInit(Value);
     // Not supported by the TSD Registry, but not an error either.
     return true;
   }
 
+  bool getDisableMemInit() const { return *getTlsPtr() & 1; }
+
 private:
   ALWAYS_INLINE uptr *getTlsPtr() const {
 #if SCUDO_HAS_PLATFORM_TLS_SLOT
     return reinterpret_cast<uptr *>(getPlatformAllocatorTlsSlot());
 #else
     static thread_local uptr ThreadTSD;
     return &ThreadTSD;
 #endif
   }
 
+  static_assert(alignof(TSD<Allocator>) >= 2, "");
+
   ALWAYS_INLINE void setCurrentTSD(TSD<Allocator> *CurrentTSD) {
-    *getTlsPtr() = reinterpret_cast<uptr>(CurrentTSD);
+    *getTlsPtr() &= 1;
+    *getTlsPtr() |= reinterpret_cast<uptr>(CurrentTSD);
   }
 
   ALWAYS_INLINE TSD<Allocator> *getCurrentTSD() {
-    return reinterpret_cast<TSD<Allocator> *>(*getTlsPtr());
+    return reinterpret_cast<TSD<Allocator> *>(*getTlsPtr() & ~1ULL);
   }
 
   bool setNumberOfTSDs(u32 N) {
     ScopedLock L(MutexTSDs);
     if (N < NumberOfTSDs)
       return false;
     if (N > TSDsArraySize)
       N = TSDsArraySize;
for (u32 I = 0; I < N; I++) {
         B = T % B;
       }
       if (A == 1)
         CoPrimes[NumberOfCoPrimes++] = I + 1;
     }
     return true;
   }
 
+  void setDisableMemInit(bool B) {
+    *getTlsPtr() &= ~1ULL;
+    *getTlsPtr() |= B;
+  }
+
   void initOnceMaybe(Allocator *Instance) {
     ScopedLock L(Mutex);
     if (LIKELY(Initialized))
       return;
     initLinkerInitialized(Instance); // Sets Initialized.
   }
 
   NOINLINE void initThread(Allocator *Instance) {

compiler-rt/lib/scudo/standalone/wrappers_c.inc

 //===-- wrappers_c.inc ------------------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef SCUDO_PREFIX
 #error "Define SCUDO_PREFIX prior to including this file!"
 #endif
 
 // malloc-type functions have to be aligned to std::max_align_t. This is
 // distinct from (1U << SCUDO_MIN_ALIGNMENT_LOG), since C++ new-type functions
 // do not have to abide by the same requirement.
 #ifndef SCUDO_MALLOC_ALIGNMENT
 #define SCUDO_MALLOC_ALIGNMENT FIRST_32_SECOND_64(8U, 16U)
 #endif
 
 extern "C" {
 
 INTERFACE WEAK void *SCUDO_PREFIX(calloc)(size_t nmemb, size_t size) {
   scudo::uptr Product;
   if (UNLIKELY(scudo::checkForCallocOverflow(size, nmemb, &Product))) {
     if (SCUDO_ALLOCATOR.canReturnNull()) {
       errno = ENOMEM;
       return nullptr;
     }
     scudo::reportCallocOverflow(nmemb, size);
   }
   return scudo::setErrnoOnNull(SCUDO_ALLOCATOR.allocate(
       Product, scudo::Chunk::Origin::Malloc, SCUDO_MALLOC_ALIGNMENT, true));
 }
 
 INTERFACE WEAK void SCUDO_PREFIX(free)(void *ptr) {
   SCUDO_ALLOCATOR.deallocate(ptr, scudo::Chunk::Origin::Malloc);
 }
 
 INTERFACE WEAK struct SCUDO_MALLINFO SCUDO_PREFIX(mallinfo)(void) {
if (param == M_DECAY_TIME) {
     SCUDO_ALLOCATOR.releaseToOS();
     return 1;
   } else {
     scudo::Option option;
     switch (param) {
     case M_MEMTAG_TUNING:
       option = scudo::Option::MemtagTuning;
       break;
+    case M_THREAD_DISABLE_MEM_INIT:
+      option = scudo::Option::ThreadDisableMemInit;
+      break;
     case M_CACHE_COUNT_MAX:
       option = scudo::Option::MaxCacheEntriesCount;
       break;
     case M_CACHE_SIZE_MAX:
       option = scudo::Option::MaxCacheEntrySize;
       break;
     case M_TSDS_COUNT_MAX:
       option = scudo::Option::MaxTSDsCount;