diff --git a/compiler-rt/lib/scudo/standalone/combined.h b/compiler-rt/lib/scudo/standalone/combined.h --- a/compiler-rt/lib/scudo/standalone/combined.h +++ b/compiler-rt/lib/scudo/standalone/combined.h @@ -100,7 +100,7 @@ // Reset tag to 0 as this chunk may have been previously used for a tagged // user allocation. - if (UNLIKELY(Allocator.useMemoryTagging())) + if (UNLIKELY(useMemoryTagging(Allocator.Primary.Options.load()))) storeTags(reinterpret_cast(Ptr), reinterpret_cast(Ptr) + sizeof(QuarantineBatch)); @@ -161,6 +161,9 @@ Primary.Options.set(OptionBit::DeallocTypeMismatch); if (getFlags()->delete_size_mismatch) Primary.Options.set(OptionBit::DeleteSizeMismatch); + if (allocatorSupportsMemoryTagging() && + systemSupportsMemoryTagging()) + Primary.Options.set(OptionBit::UseMemoryTagging); Primary.Options.set(OptionBit::UseOddEvenTags); QuarantineMaxChunkSize = @@ -240,7 +243,7 @@ } ALWAYS_INLINE void *untagPointerMaybe(void *Ptr) { - if (Primary.SupportsMemoryTagging) + if (allocatorSupportsMemoryTagging()) return reinterpret_cast( untagPointer(reinterpret_cast(Ptr))); return Ptr; @@ -367,7 +370,7 @@ // // When memory tagging is enabled, zeroing the contents is done as part of // setting the tag. - if (UNLIKELY(useMemoryTagging(Options))) { + if (UNLIKELY(useMemoryTagging(Options))) { uptr PrevUserPtr; Chunk::UnpackedHeader Header; const uptr BlockSize = PrimaryT::getSizeByClassId(ClassId); @@ -594,7 +597,7 @@ : BlockEnd - (reinterpret_cast(OldPtr) + NewSize)) & Chunk::SizeOrUnusedBytesMask; Chunk::compareExchangeHeader(Cookie, OldPtr, &NewHeader, &OldHeader); - if (UNLIKELY(ClassId && useMemoryTagging(Options))) { + if (UNLIKELY(ClassId && useMemoryTagging(Options))) { resizeTaggedChunk(reinterpret_cast(OldTaggedPtr) + OldSize, reinterpret_cast(OldTaggedPtr) + NewSize, BlockEnd); @@ -692,7 +695,7 @@ if (getChunkFromBlock(Block, &Chunk, &Header) && Header.State == Chunk::State::Allocated) { uptr TaggedChunk = Chunk; - if (useMemoryTagging(Primary.Options.load())) + if (useMemoryTagging(Primary.Options.load())) TaggedChunk = loadTag(Chunk); Callback(TaggedChunk, getSize(reinterpret_cast(Chunk), &Header), Arg); @@ -783,15 +786,14 @@ Header.State == Chunk::State::Allocated; } - bool useMemoryTagging() const { - return useMemoryTagging(Primary.Options.load()); + bool useMemoryTaggingTestOnly() const { + return useMemoryTagging(Primary.Options.load()); } - static bool useMemoryTagging(Options Options) { - return PrimaryT::useMemoryTagging(Options); + void disableMemoryTagging() { + if (allocatorSupportsMemoryTagging()) + Primary.Options.clear(OptionBit::UseMemoryTagging); } - void disableMemoryTagging() { Primary.disableMemoryTagging(); } - void setTrackAllocationStacks(bool Track) { initThreadMaybe(); if (Track) @@ -823,7 +825,7 @@ const char *MemoryTags, uintptr_t MemoryAddr, size_t MemorySize) { *ErrorInfo = {}; - if (!PrimaryT::SupportsMemoryTagging || + if (!allocatorSupportsMemoryTagging() || MemoryAddr + MemorySize < MemoryAddr) return; @@ -942,7 +944,7 @@ static_assert(MinAlignment >= sizeof(Chunk::PackedHeader), "Minimal alignment must at least cover a chunk header."); - static_assert(!PrimaryT::SupportsMemoryTagging || + static_assert(!allocatorSupportsMemoryTagging() || MinAlignment >= archMemoryTagGranuleSize(), ""); @@ -1037,7 +1039,7 @@ void quarantineOrDeallocateChunk(Options Options, void *Ptr, Chunk::UnpackedHeader *Header, uptr Size) { Chunk::UnpackedHeader NewHeader = *Header; - if (UNLIKELY(NewHeader.ClassId && useMemoryTagging(Options))) { + if (UNLIKELY(NewHeader.ClassId && useMemoryTagging(Options))) { u8 PrevTag = extractTag(loadTag(reinterpret_cast(Ptr))); if (!TSDRegistry.getDisableMemInit()) { uptr TaggedBegin, TaggedEnd; diff --git a/compiler-rt/lib/scudo/standalone/options.h b/compiler-rt/lib/scudo/standalone/options.h --- a/compiler-rt/lib/scudo/standalone/options.h +++ b/compiler-rt/lib/scudo/standalone/options.h @@ -11,6 +11,7 @@ #include "atomic_helpers.h" #include "common.h" +#include "memtag.h" namespace scudo { @@ -36,6 +37,11 @@ } }; +template bool useMemoryTagging(Options Options) { + return allocatorSupportsMemoryTagging() && + Options.get(OptionBit::UseMemoryTagging); +} + struct AtomicOptions { atomic_u32 Val; diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h --- a/compiler-rt/lib/scudo/standalone/primary32.h +++ b/compiler-rt/lib/scudo/standalone/primary32.h @@ -50,7 +50,6 @@ typedef SizeClassAllocator32 ThisT; typedef SizeClassAllocatorLocalCache CacheT; typedef typename CacheT::TransferBatch TransferBatch; - static const bool SupportsMemoryTagging = false; static uptr getSizeByClassId(uptr ClassId) { return (ClassId == SizeClassMap::BatchClassId) @@ -216,9 +215,6 @@ return TotalReleasedBytes; } - static bool useMemoryTagging(UNUSED Options Options) { return false; } - void disableMemoryTagging() {} - const char *getRegionInfoArrayAddress() const { return nullptr; } static uptr getRegionInfoArraySize() { return 0; } diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h --- a/compiler-rt/lib/scudo/standalone/primary64.h +++ b/compiler-rt/lib/scudo/standalone/primary64.h @@ -46,8 +46,6 @@ typedef SizeClassAllocator64 ThisT; typedef SizeClassAllocatorLocalCache CacheT; typedef typename CacheT::TransferBatch TransferBatch; - static const bool SupportsMemoryTagging = - allocatorSupportsMemoryTagging(); static uptr getSizeByClassId(uptr ClassId) { return (ClassId == SizeClassMap::BatchClassId) @@ -76,9 +74,6 @@ Region->ReleaseInfo.LastReleaseAtNs = Time; } setOption(Option::ReleaseInterval, static_cast(ReleaseToOsInterval)); - - if (SupportsMemoryTagging && systemSupportsMemoryTagging()) - Options.set(OptionBit::UseMemoryTagging); } void init(s32 ReleaseToOsInterval) { memset(this, 0, sizeof(*this)); @@ -193,11 +188,6 @@ return TotalReleasedBytes; } - static bool useMemoryTagging(Options Options) { - return SupportsMemoryTagging && Options.get(OptionBit::UseMemoryTagging); - } - void disableMemoryTagging() { Options.clear(OptionBit::UseMemoryTagging); } - const char *getRegionInfoArrayAddress() const { return reinterpret_cast(RegionInfoArray); } @@ -335,7 +325,7 @@ if (!map(reinterpret_cast(RegionBeg + MappedUser), UserMapSize, "scudo:primary", MAP_ALLOWNOMEM | MAP_RESIZABLE | - (useMemoryTagging(Options.load()) ? MAP_MEMTAG : 0), + (useMemoryTagging(Options.load()) ? MAP_MEMTAG : 0), &Region->Data)) return nullptr; Region->MappedUser += UserMapSize; diff --git a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp --- a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp @@ -47,7 +47,7 @@ template bool isTaggedAllocation(AllocatorT *Allocator, scudo::uptr Size, scudo::uptr Alignment) { - return Allocator->useMemoryTagging() && + return Allocator->useMemoryTaggingTestOnly() && scudo::systemDetectsMemoryTagFaultsTestOnly() && isPrimaryAllocation(Size, Alignment); } @@ -162,7 +162,7 @@ for (scudo::uptr I = 0; I < Size; I++) { unsigned char V = (reinterpret_cast(P))[I]; if (isPrimaryAllocation(Size, 1U << MinAlignLog) && - !Allocator->useMemoryTagging()) + !Allocator->useMemoryTaggingTestOnly()) ASSERT_EQ(V, scudo::PatternFillByte); else ASSERT_TRUE(V == scudo::PatternFillByte || V == 0); @@ -248,7 +248,7 @@ Allocator->releaseToOS(); - if (Allocator->useMemoryTagging() && + if (Allocator->useMemoryTaggingTestOnly() && scudo::systemDetectsMemoryTagFaultsTestOnly()) { // Check that use-after-free is detected. for (scudo::uptr SizeLog = 0U; SizeLog <= 20U; SizeLog++) { @@ -493,7 +493,7 @@ using SizeClassMap = AllocatorT::PrimaryT::SizeClassMap; auto Allocator = std::unique_ptr(new AllocatorT()); - if (!Allocator->useMemoryTagging()) + if (!Allocator->useMemoryTaggingTestOnly()) return; auto CheckOddEven = [](scudo::uptr P1, scudo::uptr P2) {