diff --git a/compiler-rt/lib/scudo/standalone/allocator_config.h b/compiler-rt/lib/scudo/standalone/allocator_config.h --- a/compiler-rt/lib/scudo/standalone/allocator_config.h +++ b/compiler-rt/lib/scudo/standalone/allocator_config.h @@ -39,6 +39,7 @@ typedef MapAllocatorCache SecondaryCache; static const u32 SecondaryCacheEntriesArraySize = 32U; + static const u32 SecondaryCacheQuarantineSize = 0U; static const u32 SecondaryCacheDefaultMaxEntriesCount = 32U; static const uptr SecondaryCacheDefaultMaxEntrySize = 1UL << 19; static const s32 SecondaryCacheMinReleaseToOsIntervalMs = INT32_MIN; @@ -63,6 +64,7 @@ typedef MapAllocatorCache SecondaryCache; static const u32 SecondaryCacheEntriesArraySize = 256U; + static const u32 SecondaryCacheQuarantineSize = 32U; static const u32 SecondaryCacheDefaultMaxEntriesCount = 32U; static const uptr SecondaryCacheDefaultMaxEntrySize = 2UL << 20; static const s32 SecondaryCacheMinReleaseToOsIntervalMs = 0; @@ -88,6 +90,7 @@ typedef MapAllocatorCache SecondaryCache; static const u32 SecondaryCacheEntriesArraySize = 16U; + static const u32 SecondaryCacheQuarantineSize = 32U; static const u32 SecondaryCacheDefaultMaxEntriesCount = 4U; static const uptr SecondaryCacheDefaultMaxEntrySize = 1UL << 18; static const s32 SecondaryCacheMinReleaseToOsIntervalMs = 0; diff --git a/compiler-rt/lib/scudo/standalone/combined.h b/compiler-rt/lib/scudo/standalone/combined.h --- a/compiler-rt/lib/scudo/standalone/combined.h +++ b/compiler-rt/lib/scudo/standalone/combined.h @@ -71,12 +71,10 @@ NewHeader.State = Chunk::State::Available; Chunk::compareExchangeHeader(Allocator.Cookie, Ptr, &NewHeader, &Header); + if (allocatorSupportsMemoryTagging()) + Ptr = untagPointer(Ptr); void *BlockBegin = Allocator::getBlockBegin(Ptr, &NewHeader); - const uptr ClassId = NewHeader.ClassId; - if (LIKELY(ClassId)) - Cache.deallocate(ClassId, BlockBegin); - else - Allocator.Secondary.deallocate(BlockBegin); + Cache.deallocate(NewHeader.ClassId, BlockBegin); } // We take a shortcut when allocating a quarantine batch by working with the @@ -243,11 +241,26 @@ TSD->Cache.destroy(&Stats); } - ALWAYS_INLINE void *untagPointerMaybe(void *Ptr) { - if (allocatorSupportsMemoryTagging()) - return reinterpret_cast( - untagPointer(reinterpret_cast(Ptr))); - return Ptr; + ALWAYS_INLINE void *getHeaderTaggedPointer(void *Ptr) { + if (!allocatorSupportsMemoryTagging()) + return Ptr; + auto UntaggedPtr = untagPointer(Ptr); + if (UntaggedPtr != Ptr) + return UntaggedPtr; + // Secondary, or pointer allocated while memory tagging is unsupported or + // disabled. The tag mismatch is okay in the latter case because tags will + // not be checked. + return addHeaderTag(Ptr); + } + + ALWAYS_INLINE uptr addHeaderTag(uptr Ptr) { + if (!allocatorSupportsMemoryTagging()) + return Ptr; + return addFixedTag(Ptr, 2); + } + + ALWAYS_INLINE void *addHeaderTag(void *Ptr) { + return reinterpret_cast(addHeaderTag(reinterpret_cast(Ptr))); } NOINLINE u32 collectStackTrace() { @@ -344,7 +357,7 @@ TSD->unlock(); } if (UNLIKELY(ClassId == 0)) - Block = Secondary.allocate(NeededSize, Alignment, &SecondaryBlockEnd, + Block = Secondary.allocate(Options, Size, Alignment, &SecondaryBlockEnd, FillContents); if (UNLIKELY(!Block)) { @@ -440,12 +453,21 @@ TaggedPtr = prepareTaggedChunk(Ptr, Size, OddEvenMask, BlockEnd); } storeAllocationStackMaybe(Options, Ptr); - } else if (UNLIKELY(FillContents != NoFill)) { - // This condition is not necessarily unlikely, but since memset is - // costly, we might as well mark it as such. - memset(Block, FillContents == ZeroFill ? 0 : PatternFillByte, - PrimaryT::getSizeByClassId(ClassId)); + } else { + Block = addHeaderTag(Block); + Ptr = addHeaderTag(Ptr); + if (UNLIKELY(FillContents != NoFill)) { + // This condition is not necessarily unlikely, but since memset is + // costly, we might as well mark it as such. + memset(Block, FillContents == ZeroFill ? 0 : PatternFillByte, + PrimaryT::getSizeByClassId(ClassId)); + } } + } else { + Block = addHeaderTag(Block); + Ptr = addHeaderTag(Ptr); + if (UNLIKELY(useMemoryTagging(Options))) + storeTags(reinterpret_cast(Block), reinterpret_cast(Ptr)); } Chunk::UnpackedHeader Header = {}; @@ -499,7 +521,7 @@ if (UNLIKELY(!isAligned(reinterpret_cast(Ptr), MinAlignment))) reportMisalignedPointer(AllocatorAction::Deallocating, Ptr); - Ptr = untagPointerMaybe(Ptr); + Ptr = getHeaderTaggedPointer(Ptr); Chunk::UnpackedHeader Header; Chunk::loadHeader(Cookie, Ptr, &Header); @@ -538,7 +560,7 @@ } void *OldTaggedPtr = OldPtr; - OldPtr = untagPointerMaybe(OldPtr); + OldPtr = getHeaderTaggedPointer(OldPtr); // The following cases are handled by the C wrappers. DCHECK_NE(OldPtr, nullptr); @@ -574,7 +596,7 @@ Chunk::Origin::Malloc); } - void *BlockBegin = getBlockBegin(OldPtr, &OldHeader); + void *BlockBegin = getBlockBegin(OldTaggedPtr, &OldHeader); uptr BlockEnd; uptr OldSize; const uptr ClassId = OldHeader.ClassId; @@ -584,18 +606,19 @@ OldSize = OldHeader.SizeOrUnusedBytes; } else { BlockEnd = SecondaryT::getBlockEnd(BlockBegin); - OldSize = BlockEnd - - (reinterpret_cast(OldPtr) + OldHeader.SizeOrUnusedBytes); + OldSize = BlockEnd - (reinterpret_cast(OldTaggedPtr) + + OldHeader.SizeOrUnusedBytes); } // If the new chunk still fits in the previously allocated block (with a // reasonable delta), we just keep the old block, and update the chunk // header to reflect the size change. - if (reinterpret_cast(OldPtr) + NewSize <= BlockEnd) { + if (reinterpret_cast(OldTaggedPtr) + NewSize <= BlockEnd) { if (NewSize > OldSize || (OldSize - NewSize) < getPageSizeCached()) { Chunk::UnpackedHeader NewHeader = OldHeader; NewHeader.SizeOrUnusedBytes = (ClassId ? NewSize - : BlockEnd - (reinterpret_cast(OldPtr) + NewSize)) & + : BlockEnd - + (reinterpret_cast(OldTaggedPtr) + NewSize)) & Chunk::SizeOrUnusedBytesMask; Chunk::compareExchangeHeader(Cookie, OldPtr, &NewHeader, &OldHeader); if (UNLIKELY(ClassId && useMemoryTagging(Options))) { @@ -688,14 +711,30 @@ initThreadMaybe(); const uptr From = Base; const uptr To = Base + Size; - auto Lambda = [this, From, To, Callback, Arg](uptr Block) { + bool MayHaveTaggedPrimary = allocatorSupportsMemoryTagging() && + systemSupportsMemoryTagging(); + auto Lambda = [this, From, To, MayHaveTaggedPrimary, Callback, + Arg](uptr Block) { if (Block < From || Block >= To) return; uptr Chunk; Chunk::UnpackedHeader Header; - if (getChunkFromBlock(Block, &Chunk, &Header) && - Header.State == Chunk::State::Allocated) { + if (MayHaveTaggedPrimary) { + // A chunk header can either have a zero tag (tagged primary) or the + // header tag (secondary, or untagged primary). We don't know which so + // try both. + ScopedDisableMemoryTagChecks x; + if (!getChunkFromBlock(Block, &Chunk, &Header) && + !getChunkFromBlock(addHeaderTag(Block), &Chunk, &Header)) + return; + } else { + if (!getChunkFromBlock(addHeaderTag(Block), &Chunk, &Header)) + return; + } + if (Header.State == Chunk::State::Allocated) { uptr TaggedChunk = Chunk; + if (allocatorSupportsMemoryTagging()) + TaggedChunk = untagPointer(TaggedChunk); if (useMemoryTagging(Primary.Options.load())) TaggedChunk = loadTag(Chunk); Callback(TaggedChunk, getSize(reinterpret_cast(Chunk), &Header), @@ -756,7 +795,7 @@ return GuardedAlloc.getSize(Ptr); #endif // GWP_ASAN_HOOKS - Ptr = untagPointerMaybe(const_cast(Ptr)); + Ptr = getHeaderTaggedPointer(const_cast(Ptr)); Chunk::UnpackedHeader Header; Chunk::loadHeader(Cookie, Ptr, &Header); // Getting the usable size of a chunk only makes sense if it's allocated. @@ -781,7 +820,7 @@ #endif // GWP_ASAN_HOOKS if (!Ptr || !isAligned(reinterpret_cast(Ptr), MinAlignment)) return false; - Ptr = untagPointerMaybe(const_cast(Ptr)); + Ptr = getHeaderTaggedPointer(const_cast(Ptr)); Chunk::UnpackedHeader Header; return Chunk::isValid(Cookie, Ptr, &Header) && Header.State == Chunk::State::Allocated; @@ -791,8 +830,11 @@ return useMemoryTagging(Primary.Options.load()); } void disableMemoryTagging() { - if (allocatorSupportsMemoryTagging()) + initThreadMaybe(); + if (allocatorSupportsMemoryTagging()) { + Secondary.disableMemoryTagging(); Primary.Options.clear(OptionBit::UseMemoryTagging); + } } void setTrackAllocationStacks(bool Track) { @@ -1033,6 +1075,8 @@ const uptr SizeOrUnusedBytes = Header->SizeOrUnusedBytes; if (LIKELY(Header->ClassId)) return SizeOrUnusedBytes; + if (allocatorSupportsMemoryTagging()) + Ptr = untagPointer(const_cast(Ptr)); return SecondaryT::getBlockEnd(getBlockBegin(Ptr, Header)) - reinterpret_cast(Ptr) - SizeOrUnusedBytes; } @@ -1058,11 +1102,14 @@ // If the quarantine is disabled, the actual size of a chunk is 0 or larger // than the maximum allowed, we return a chunk directly to the backend. // This purposefully underflows for Size == 0. - const bool BypassQuarantine = - !Quarantine.getCacheSize() || ((Size - 1) >= QuarantineMaxChunkSize); + const bool BypassQuarantine = !Quarantine.getCacheSize() || + ((Size - 1) >= QuarantineMaxChunkSize) || + !NewHeader.ClassId; if (BypassQuarantine) { NewHeader.State = Chunk::State::Available; Chunk::compareExchangeHeader(Cookie, Ptr, &NewHeader, Header); + if (allocatorSupportsMemoryTagging()) + Ptr = untagPointer(Ptr); void *BlockBegin = getBlockBegin(Ptr, &NewHeader); const uptr ClassId = NewHeader.ClassId; if (LIKELY(ClassId)) { @@ -1072,7 +1119,10 @@ if (UnlockRequired) TSD->unlock(); } else { - Secondary.deallocate(BlockBegin); + if (UNLIKELY(useMemoryTagging(Options))) + storeTags(reinterpret_cast(BlockBegin), + reinterpret_cast(Ptr)); + Secondary.deallocate(Options, BlockBegin); } } else { NewHeader.State = Chunk::State::Quarantined; diff --git a/compiler-rt/lib/scudo/standalone/common.h b/compiler-rt/lib/scudo/standalone/common.h --- a/compiler-rt/lib/scudo/standalone/common.h +++ b/compiler-rt/lib/scudo/standalone/common.h @@ -165,6 +165,9 @@ void unmap(void *Addr, uptr Size, uptr Flags = 0, MapPlatformData *Data = nullptr); +void setMemoryPermission(uptr Addr, uptr Size, uptr Flags, + MapPlatformData *Data = nullptr); + void releasePagesToOS(uptr BaseAddress, uptr Offset, uptr Size, MapPlatformData *Data = nullptr); diff --git a/compiler-rt/lib/scudo/standalone/fuchsia.cpp b/compiler-rt/lib/scudo/standalone/fuchsia.cpp --- a/compiler-rt/lib/scudo/standalone/fuchsia.cpp +++ b/compiler-rt/lib/scudo/standalone/fuchsia.cpp @@ -135,6 +135,16 @@ } } +void setMemoryPermission(UNUSED uptr Addr, UNUSED uptr Size, UNUSED uptr Flags, + UNUSED MapPlatformData *Data) { + const zx_vm_option_t Prot = + (Flags & MAP_NOACCESS) ? 0 : (ZX_VM_PERM_READ | ZX_VM_PERM_WRITE); + DCHECK(Data); + DCHECK_NE(Data->Vmar, ZX_HANDLE_INVALID); + if (_zx_vmar_protect(Data->Vmar, Prot, Addr, Size) != ZX_OK) + dieOnMapUnmapError(); +} + void releasePagesToOS(UNUSED uptr BaseAddress, uptr Offset, uptr Size, MapPlatformData *Data) { DCHECK(Data); diff --git a/compiler-rt/lib/scudo/standalone/linux.cpp b/compiler-rt/lib/scudo/standalone/linux.cpp --- a/compiler-rt/lib/scudo/standalone/linux.cpp +++ b/compiler-rt/lib/scudo/standalone/linux.cpp @@ -50,14 +50,14 @@ MmapProt = PROT_NONE; } else { MmapProt = PROT_READ | PROT_WRITE; + } #if defined(__aarch64__) #ifndef PROT_MTE #define PROT_MTE 0x20 #endif - if (Flags & MAP_MEMTAG) - MmapProt |= PROT_MTE; + if (Flags & MAP_MEMTAG) + MmapProt |= PROT_MTE; #endif - } if (Addr) { // Currently no scenario for a noaccess mapping with a fixed address. DCHECK_EQ(Flags & MAP_NOACCESS, 0); @@ -70,7 +70,7 @@ return nullptr; } #if SCUDO_ANDROID - if (!(Flags & MAP_NOACCESS)) + if (Name) prctl(ANDROID_PR_SET_VMA, ANDROID_PR_SET_VMA_ANON_NAME, P, Size, Name); #endif return P; @@ -82,6 +82,13 @@ dieOnMapUnmapError(); } +void setMemoryPermission(uptr Addr, uptr Size, uptr Flags, + UNUSED MapPlatformData *Data) { + int Prot = (Flags & MAP_NOACCESS) ? PROT_NONE : (PROT_READ | PROT_WRITE); + if (mprotect(reinterpret_cast(Addr), Size, Prot) != 0) + dieOnMapUnmapError(); +} + void releasePagesToOS(uptr BaseAddress, uptr Offset, uptr Size, UNUSED MapPlatformData *Data) { void *Addr = reinterpret_cast(BaseAddress + Offset); diff --git a/compiler-rt/lib/scudo/standalone/memtag.h b/compiler-rt/lib/scudo/standalone/memtag.h --- a/compiler-rt/lib/scudo/standalone/memtag.h +++ b/compiler-rt/lib/scudo/standalone/memtag.h @@ -23,7 +23,15 @@ #if defined(__aarch64__) || defined(SCUDO_FUZZ) +// We assume that Top-Byte Ignore is enabled if the architecture supports memory +// tagging. Not all operating systems enable TBI, so we only claim architectural +// support for memory tagging if the operating system enables TBI. +#if SCUDO_LINUX inline constexpr bool archSupportsMemoryTagging() { return true; } +#else +inline constexpr bool archSupportsMemoryTagging() { return false; } +#endif + inline constexpr uptr archMemoryTagGranuleSize() { return 16; } inline uptr untagPointer(uptr Ptr) { return Ptr & ((1ULL << 56) - 1); } @@ -118,6 +126,8 @@ return TaggedPtr; } +inline uptr addFixedTag(uptr Ptr, uptr Tag) { return Ptr | (Tag << 56); } + inline uptr storeTags(uptr Begin, uptr End) { DCHECK(Begin % 16 == 0); if (Begin != End) { @@ -243,6 +253,12 @@ UNREACHABLE("memory tagging not supported"); } +inline uptr addFixedTag(uptr Ptr, uptr Tag) { + (void)Ptr; + (void)Tag; + UNREACHABLE("memory tagging not supported"); +} + inline uptr storeTags(uptr Begin, uptr End) { (void)Begin; (void)End; @@ -278,6 +294,10 @@ *TaggedEnd = storeTags(*TaggedBegin, *TaggedBegin + Size); } +inline void *untagPointer(void *Ptr) { + return reinterpret_cast(untagPointer(reinterpret_cast(Ptr))); +} + template inline constexpr bool allocatorSupportsMemoryTagging() { return archSupportsMemoryTagging() && Config::MaySupportMemoryTagging; diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h --- a/compiler-rt/lib/scudo/standalone/primary64.h +++ b/compiler-rt/lib/scudo/standalone/primary64.h @@ -58,7 +58,7 @@ void initLinkerInitialized(s32 ReleaseToOsInterval) { // Reserve the space required for the Primary. PrimaryBase = reinterpret_cast( - map(nullptr, PrimarySize, "scudo:primary", MAP_NOACCESS, &Data)); + map(nullptr, PrimarySize, nullptr, MAP_NOACCESS, &Data)); u32 Seed; const u64 Time = getMonotonicTime(); diff --git a/compiler-rt/lib/scudo/standalone/secondary.h b/compiler-rt/lib/scudo/standalone/secondary.h --- a/compiler-rt/lib/scudo/standalone/secondary.h +++ b/compiler-rt/lib/scudo/standalone/secondary.h @@ -28,7 +28,8 @@ struct Header { LargeBlock::Header *Prev; LargeBlock::Header *Next; - uptr BlockEnd; + uptr CommitBase; + uptr CommitSize; uptr MapBase; uptr MapSize; [[no_unique_address]] MapPlatformData Data; @@ -38,29 +39,42 @@ return roundUpTo(sizeof(Header), 1U << SCUDO_MIN_ALIGNMENT_LOG); } -static Header *getHeader(uptr Ptr) { - return reinterpret_cast
(Ptr - getHeaderSize()); +template static uptr addHeaderTag(uptr Ptr) { + if (allocatorSupportsMemoryTagging()) + return addFixedTag(Ptr, 1); + return Ptr; } -static Header *getHeader(const void *Ptr) { - return getHeader(reinterpret_cast(Ptr)); +template static Header *getHeader(uptr Ptr) { + return reinterpret_cast
(addHeaderTag(Ptr) - + getHeaderSize()); +} + +template static Header *getHeader(const void *Ptr) { + return getHeader(reinterpret_cast(Ptr)); } } // namespace LargeBlock +static void unmap(LargeBlock::Header *H) { + MapPlatformData Data = H->Data; + unmap(reinterpret_cast(H->MapBase), H->MapSize, UNMAP_ALL, &Data); +} + class MapAllocatorNoCache { public: void initLinkerInitialized(UNUSED s32 ReleaseToOsInterval) {} void init(UNUSED s32 ReleaseToOsInterval) {} - bool retrieve(UNUSED uptr Size, UNUSED LargeBlock::Header **H, - UNUSED bool *Zeroed) { + bool retrieve(UNUSED Options Options, UNUSED uptr Size, UNUSED uptr Alignment, + UNUSED LargeBlock::Header **H, UNUSED bool *Zeroed) { return false; } - bool store(UNUSED LargeBlock::Header *H) { return false; } + void store(UNUSED Options Options, UNUSED LargeBlock::Header *H) { unmap(H); } bool canCache(UNUSED uptr Size) { return false; } void disable() {} void enable() {} void releaseToOS() {} + void disableMemoryTagging() {} bool setOption(Option O, UNUSED sptr Value) { if (O == Option::ReleaseInterval || O == Option::MaxCacheEntriesCount || O == Option::MaxCacheEntrySize) @@ -89,67 +103,137 @@ initLinkerInitialized(ReleaseToOsInterval); } - bool store(LargeBlock::Header *H) { + void store(Options Options, LargeBlock::Header *H) { + if (!canCache(H->CommitSize)) + return unmap(H); + bool EntryCached = false; bool EmptyCache = false; + const s32 Interval = atomic_load_relaxed(&ReleaseToOsIntervalMs); const u64 Time = getMonotonicTime(); const u32 MaxCount = atomic_load_relaxed(&MaxEntriesCount); - { + CachedBlock Entry; + Entry.CommitBase = H->CommitBase; + Entry.CommitSize = H->CommitSize; + Entry.MapBase = H->MapBase; + Entry.MapSize = H->MapSize; + Entry.BlockBegin = reinterpret_cast(H + 1); + Entry.Data = H->Data; + Entry.Time = Time; + if (useMemoryTagging(Options)) { + if (Interval == 0 && !SCUDO_FUCHSIA) { + // Release the memory and make it inaccessible at the same time by + // creating a new MAP_NOACCESS mapping on top of the existing mapping. + // Fuchsia does not support replacing mappings by creating a new mapping + // on top so we just do the two syscalls there. + Entry.Time = 0; + map(reinterpret_cast(Entry.CommitBase), Entry.CommitSize, + "scudo:secondary", MAP_RESIZABLE | MAP_NOACCESS | MAP_MEMTAG, + &Entry.Data); + } else { + setMemoryPermission(Entry.CommitBase, Entry.CommitSize, MAP_NOACCESS, + &Entry.Data); + } + } + do { ScopedLock L(Mutex); + if (useMemoryTagging(Options) && QuarantinePos == -1U) { + // If we get here then memory tagging was disabled in between when we + // read Options and when we locked Mutex. We can't insert our entry into + // the quarantine or the cache because the permissions would be wrong so + // just unmap it. + break; + } + if (Config::SecondaryCacheQuarantineSize && + useMemoryTagging(Options)) { + QuarantinePos = + (QuarantinePos + 1) % Config::SecondaryCacheQuarantineSize; + if (!Quarantine[QuarantinePos].CommitBase) { + Quarantine[QuarantinePos] = Entry; + return; + } + CachedBlock PrevEntry = Quarantine[QuarantinePos]; + Quarantine[QuarantinePos] = Entry; + Entry = PrevEntry; + } if (EntriesCount >= MaxCount) { if (IsFullEvents++ == 4U) EmptyCache = true; } else { for (u32 I = 0; I < MaxCount; I++) { - if (Entries[I].Block) + if (Entries[I].CommitBase) continue; if (I != 0) Entries[I] = Entries[0]; - Entries[0].Block = reinterpret_cast(H); - Entries[0].BlockEnd = H->BlockEnd; - Entries[0].MapBase = H->MapBase; - Entries[0].MapSize = H->MapSize; - Entries[0].Data = H->Data; - Entries[0].Time = Time; + Entries[0] = Entry; EntriesCount++; EntryCached = true; break; } } - } - s32 Interval; + } while (0); if (EmptyCache) empty(); - else if ((Interval = atomic_load_relaxed(&ReleaseToOsIntervalMs)) >= 0) + else if (Interval >= 0) releaseOlderThan(Time - static_cast(Interval) * 1000000); - return EntryCached; + if (!EntryCached) + unmap(reinterpret_cast(Entry.MapBase), Entry.MapSize, UNMAP_ALL, + &Entry.Data); } - bool retrieve(uptr Size, LargeBlock::Header **H, bool *Zeroed) { + bool retrieve(Options Options, uptr Size, uptr Alignment, + LargeBlock::Header **H, bool *Zeroed) { const uptr PageSize = getPageSizeCached(); const u32 MaxCount = atomic_load_relaxed(&MaxEntriesCount); - ScopedLock L(Mutex); - if (EntriesCount == 0) - return false; - for (u32 I = 0; I < MaxCount; I++) { - if (!Entries[I].Block) - continue; - const uptr BlockSize = Entries[I].BlockEnd - Entries[I].Block; - if (Size > BlockSize) - continue; - if (Size < BlockSize - PageSize * 4U) - continue; - *H = reinterpret_cast(Entries[I].Block); - *Zeroed = Entries[I].Time == 0; - Entries[I].Block = 0; - (*H)->BlockEnd = Entries[I].BlockEnd; - (*H)->MapBase = Entries[I].MapBase; - (*H)->MapSize = Entries[I].MapSize; - (*H)->Data = Entries[I].Data; + bool Found = false; + CachedBlock Entry; + uptr HeaderPos; + { + ScopedLock L(Mutex); + if (EntriesCount == 0) + return false; + for (u32 I = 0; I < MaxCount; I++) { + const uptr CommitBase = Entries[I].CommitBase; + if (!CommitBase) + continue; + const uptr CommitSize = Entries[I].CommitSize; + HeaderPos = roundDownTo(CommitBase + CommitSize - Size, Alignment) - + Chunk::getHeaderSize() - LargeBlock::getHeaderSize(); + if (HeaderPos > CommitBase + CommitSize) + continue; + if (HeaderPos < CommitBase || HeaderPos > CommitBase + PageSize * 4U) + continue; + Found = true; + Entry = Entries[I]; + Entries[I].CommitBase = 0; + break; + } + } + if (Found) { + *H = reinterpret_cast( + LargeBlock::addHeaderTag(HeaderPos)); + *Zeroed = Entry.Time == 0; + if (useMemoryTagging(Options)) + setMemoryPermission(Entry.CommitBase, Entry.CommitSize, 0, &Entry.Data); + uptr NewBlockBegin = reinterpret_cast(*H + 1); + if (useMemoryTagging(Options)) { + if (*Zeroed) + storeTags(LargeBlock::addHeaderTag(Entry.CommitBase), + NewBlockBegin); + else if (Entry.BlockBegin < NewBlockBegin) + storeTags(Entry.BlockBegin, NewBlockBegin); + else + storeTags(untagPointer(NewBlockBegin), + untagPointer(Entry.BlockBegin)); + } + (*H)->CommitBase = Entry.CommitBase; + (*H)->CommitSize = Entry.CommitSize; + (*H)->MapBase = Entry.MapBase; + (*H)->MapSize = Entry.MapSize; + (*H)->Data = Entry.Data; EntriesCount--; - return true; } - return false; + return Found; } bool canCache(uptr Size) { @@ -181,6 +265,23 @@ void releaseToOS() { releaseOlderThan(UINT64_MAX); } + void disableMemoryTagging() { + ScopedLock L(Mutex); + for (u32 I = 0; I != Config::SecondaryCacheQuarantineSize; ++I) { + if (Quarantine[I].CommitBase) { + unmap(reinterpret_cast(Quarantine[I].MapBase), + Quarantine[I].MapSize, UNMAP_ALL, &Quarantine[I].Data); + Quarantine[I].CommitBase = 0; + } + } + const u32 MaxCount = atomic_load_relaxed(&MaxEntriesCount); + for (u32 I = 0; I < MaxCount; I++) + if (Entries[I].CommitBase) + setMemoryPermission(Entries[I].CommitBase, Entries[I].CommitSize, 0, + &Entries[I].Data); + QuarantinePos = -1U; + } + void disable() { Mutex.lock(); } void enable() { Mutex.unlock(); } @@ -196,12 +297,12 @@ { ScopedLock L(Mutex); for (uptr I = 0; I < Config::SecondaryCacheEntriesArraySize; I++) { - if (!Entries[I].Block) + if (!Entries[I].CommitBase) continue; MapInfo[N].MapBase = reinterpret_cast(Entries[I].MapBase); MapInfo[N].MapSize = Entries[I].MapSize; MapInfo[N].Data = Entries[I].Data; - Entries[I].Block = 0; + Entries[I].CommitBase = 0; N++; } EntriesCount = 0; @@ -212,32 +313,38 @@ &MapInfo[I].Data); } - void releaseOlderThan(u64 Time) { - ScopedLock L(Mutex); - if (!EntriesCount) - return; - for (uptr I = 0; I < Config::SecondaryCacheEntriesArraySize; I++) { - if (!Entries[I].Block || !Entries[I].Time || Entries[I].Time > Time) - continue; - releasePagesToOS(Entries[I].Block, 0, - Entries[I].BlockEnd - Entries[I].Block, - &Entries[I].Data); - Entries[I].Time = 0; - } - } - struct CachedBlock { - uptr Block; - uptr BlockEnd; + uptr CommitBase; + uptr CommitSize; uptr MapBase; uptr MapSize; + uptr BlockBegin; [[no_unique_address]] MapPlatformData Data; u64 Time; }; + void releaseIfOlderThan(CachedBlock &Entry, u64 Time) { + if (!Entry.CommitBase || !Entry.Time || Entry.Time > Time) + return; + releasePagesToOS(Entry.CommitBase, 0, Entry.CommitSize, &Entry.Data); + Entry.Time = 0; + } + + void releaseOlderThan(u64 Time) { + ScopedLock L(Mutex); + if (!EntriesCount) + return; + for (uptr I = 0; I < Config::SecondaryCacheQuarantineSize; I++) + releaseIfOlderThan(Quarantine[I], Time); + for (uptr I = 0; I < Config::SecondaryCacheEntriesArraySize; I++) + releaseIfOlderThan(Entries[I], Time); + } + HybridMutex Mutex; CachedBlock Entries[Config::SecondaryCacheEntriesArraySize]; u32 EntriesCount; + CachedBlock Quarantine[Config::SecondaryCacheQuarantineSize]; + u32 QuarantinePos; atomic_u32 MaxEntriesCount; atomic_uptr MaxEntrySize; uptr LargestSize; @@ -258,13 +365,15 @@ initLinkerInitialized(S, ReleaseToOsInterval); } - void *allocate(uptr Size, uptr AlignmentHint = 0, uptr *BlockEnd = nullptr, + void *allocate(Options Options, uptr Size, uptr AlignmentHint = 0, + uptr *BlockEnd = nullptr, FillContentsMode FillContents = NoFill); - void deallocate(void *Ptr); + void deallocate(Options Options, void *Ptr); static uptr getBlockEnd(void *Ptr) { - return LargeBlock::getHeader(Ptr)->BlockEnd; + auto *B = LargeBlock::getHeader(Ptr); + return B->CommitBase + B->CommitSize; } static uptr getBlockSize(void *Ptr) { @@ -284,8 +393,12 @@ } template void iterateOverBlocks(F Callback) const { - for (const auto &H : InUseBlocks) - Callback(reinterpret_cast(&H) + LargeBlock::getHeaderSize()); + for (const auto &H : InUseBlocks) { + uptr Ptr = reinterpret_cast(&H) + LargeBlock::getHeaderSize(); + if (allocatorSupportsMemoryTagging()) + Ptr = untagPointer(Ptr); + Callback(Ptr); + } } uptr canCache(uptr Size) { return Cache.canCache(Size); } @@ -294,6 +407,8 @@ void releaseToOS() { Cache.releaseToOS(); } + void disableMemoryTagging() { Cache.disableMemoryTagging(); } + private: typename Config::SecondaryCache Cache; @@ -319,26 +434,33 @@ // the committed memory will amount to something close to Size - AlignmentHint // (pending rounding and headers). template -void *MapAllocator::allocate(uptr Size, uptr AlignmentHint, - uptr *BlockEnd, +void *MapAllocator::allocate(Options Options, uptr Size, uptr Alignment, + uptr *BlockEndPtr, FillContentsMode FillContents) { - DCHECK_GE(Size, AlignmentHint); + Alignment = Max(Alignment, 1UL << SCUDO_MIN_ALIGNMENT_LOG); const uptr PageSize = getPageSizeCached(); - const uptr RoundedSize = - roundUpTo(Size + LargeBlock::getHeaderSize(), PageSize); - - if (AlignmentHint < PageSize && Cache.canCache(RoundedSize)) { + uptr RoundedSize = + roundUpTo(roundUpTo(Size, Alignment) + LargeBlock::getHeaderSize() + + Chunk::getHeaderSize(), + PageSize); + if (Alignment > PageSize) + RoundedSize += Alignment - PageSize; + + if (Alignment < PageSize && Cache.canCache(RoundedSize)) { LargeBlock::Header *H; bool Zeroed; - if (Cache.retrieve(RoundedSize, &H, &Zeroed)) { - if (BlockEnd) - *BlockEnd = H->BlockEnd; - void *Ptr = reinterpret_cast(reinterpret_cast(H) + - LargeBlock::getHeaderSize()); + if (Cache.retrieve(Options, Size, Alignment, &H, &Zeroed)) { + const uptr BlockEnd = H->CommitBase + H->CommitSize; + if (BlockEndPtr) + *BlockEndPtr = BlockEnd; + uptr PtrInt = reinterpret_cast(H) + LargeBlock::getHeaderSize(); + if (allocatorSupportsMemoryTagging()) + PtrInt = untagPointer(PtrInt); + void *Ptr = reinterpret_cast(PtrInt); if (FillContents && !Zeroed) memset(Ptr, FillContents == ZeroFill ? 0 : PatternFillByte, - H->BlockEnd - reinterpret_cast(Ptr)); - const uptr BlockSize = H->BlockEnd - reinterpret_cast(H); + BlockEnd - PtrInt); + const uptr BlockSize = BlockEnd - reinterpret_cast(H); { ScopedLock L(Mutex); InUseBlocks.push_back(H); @@ -353,9 +475,8 @@ MapPlatformData Data = {}; const uptr MapSize = RoundedSize + 2 * PageSize; - uptr MapBase = - reinterpret_cast(map(nullptr, MapSize, "scudo:secondary", - MAP_NOACCESS | MAP_ALLOWNOMEM, &Data)); + uptr MapBase = reinterpret_cast( + map(nullptr, MapSize, nullptr, MAP_NOACCESS | MAP_ALLOWNOMEM, &Data)); if (UNLIKELY(!MapBase)) return nullptr; uptr CommitBase = MapBase + PageSize; @@ -363,11 +484,11 @@ // In the unlikely event of alignments larger than a page, adjust the amount // of memory we want to commit, and trim the extra memory. - if (UNLIKELY(AlignmentHint >= PageSize)) { + if (UNLIKELY(Alignment >= PageSize)) { // For alignments greater than or equal to a page, the user pointer (eg: the // pointer that is returned by the C or C++ allocation APIs) ends up on a // page boundary , and our headers will live in the preceding page. - CommitBase = roundUpTo(MapBase + PageSize + 1, AlignmentHint) - PageSize; + CommitBase = roundUpTo(MapBase + PageSize + 1, Alignment) - PageSize; const uptr NewMapBase = CommitBase - PageSize; DCHECK_GE(NewMapBase, MapBase); // We only trim the extra memory on 32-bit platforms: 64-bit platforms @@ -376,9 +497,8 @@ unmap(reinterpret_cast(MapBase), NewMapBase - MapBase, 0, &Data); MapBase = NewMapBase; } - const uptr NewMapEnd = CommitBase + PageSize + - roundUpTo((Size - AlignmentHint), PageSize) + - PageSize; + const uptr NewMapEnd = + CommitBase + PageSize + roundUpTo(Size, PageSize) + PageSize; DCHECK_LE(NewMapEnd, MapEnd); if (SCUDO_WORDSIZE == 32U && NewMapEnd != MapEnd) { unmap(reinterpret_cast(NewMapEnd), MapEnd - NewMapEnd, 0, &Data); @@ -387,16 +507,24 @@ } const uptr CommitSize = MapEnd - PageSize - CommitBase; - const uptr Ptr = reinterpret_cast( - map(reinterpret_cast(CommitBase), CommitSize, "scudo:secondary", - MAP_RESIZABLE, &Data)); - LargeBlock::Header *H = reinterpret_cast(Ptr); + map(reinterpret_cast(CommitBase), CommitSize, "scudo:secondary", + MAP_RESIZABLE | (useMemoryTagging(Options) ? MAP_MEMTAG : 0), + &Data); + const uptr HeaderPos = + roundDownTo(CommitBase + CommitSize - Size, Alignment) - + Chunk::getHeaderSize() - LargeBlock::getHeaderSize(); + LargeBlock::Header *H = reinterpret_cast( + LargeBlock::addHeaderTag(HeaderPos)); + if (useMemoryTagging(Options)) + storeTags(LargeBlock::addHeaderTag(CommitBase), + reinterpret_cast(H + 1)); H->MapBase = MapBase; H->MapSize = MapEnd - MapBase; - H->BlockEnd = CommitBase + CommitSize; + H->CommitBase = CommitBase; + H->CommitSize = CommitSize; H->Data = Data; - if (BlockEnd) - *BlockEnd = CommitBase + CommitSize; + if (BlockEndPtr) + *BlockEndPtr = CommitBase + CommitSize; { ScopedLock L(Mutex); InUseBlocks.push_back(H); @@ -407,13 +535,13 @@ Stats.add(StatAllocated, CommitSize); Stats.add(StatMapped, H->MapSize); } - return reinterpret_cast(Ptr + LargeBlock::getHeaderSize()); + return reinterpret_cast(HeaderPos + LargeBlock::getHeaderSize()); } -template void MapAllocator::deallocate(void *Ptr) { - LargeBlock::Header *H = LargeBlock::getHeader(Ptr); - const uptr Block = reinterpret_cast(H); - const uptr CommitSize = H->BlockEnd - Block; +template +void MapAllocator::deallocate(Options Options, void *Ptr) { + LargeBlock::Header *H = LargeBlock::getHeader(Ptr); + const uptr CommitSize = H->CommitSize; { ScopedLock L(Mutex); InUseBlocks.remove(H); @@ -422,12 +550,7 @@ Stats.sub(StatAllocated, CommitSize); Stats.sub(StatMapped, H->MapSize); } - if (Cache.canCache(CommitSize) && Cache.store(H)) - return; - void *Addr = reinterpret_cast(H->MapBase); - const uptr Size = H->MapSize; - MapPlatformData Data = H->Data; - unmap(Addr, Size, UNMAP_ALL, &Data); + Cache.store(Options, H); } template diff --git a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp --- a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp @@ -44,39 +44,37 @@ return AllocatorT::PrimaryT::canAllocate(NeededSize); } -template -bool isTaggedAllocation(AllocatorT *Allocator, scudo::uptr Size, - scudo::uptr Alignment) { - return Allocator->useMemoryTaggingTestOnly() && - scudo::systemDetectsMemoryTagFaultsTestOnly() && - isPrimaryAllocation(Size, Alignment); -} - template void checkMemoryTaggingMaybe(AllocatorT *Allocator, void *P, scudo::uptr Size, scudo::uptr Alignment) { - if (!isTaggedAllocation(Allocator, Size, Alignment)) - return; - - Size = scudo::roundUpTo(Size, scudo::archMemoryTagGranuleSize()); - EXPECT_DEATH( - { - disableDebuggerdMaybe(); - reinterpret_cast(P)[-1] = 0xaa; - }, - ""); - EXPECT_DEATH( - { - disableDebuggerdMaybe(); - reinterpret_cast(P)[Size] = 0xaa; - }, - ""); + const scudo::uptr MinAlignment = 1UL << SCUDO_MIN_ALIGNMENT_LOG; + Size = scudo::roundUpTo(Size, MinAlignment); + if (Allocator->useMemoryTaggingTestOnly()) + EXPECT_DEATH( + { + disableDebuggerdMaybe(); + reinterpret_cast(P)[-1] = 0xaa; + }, + ""); + if (isPrimaryAllocation(Size, Alignment) + ? Allocator->useMemoryTaggingTestOnly() + : Alignment == MinAlignment) { + EXPECT_DEATH( + { + disableDebuggerdMaybe(); + reinterpret_cast(P)[Size] = 0xaa; + }, + ""); + } } template struct TestAllocator : scudo::Allocator { TestAllocator() { this->reset(); this->initThreadMaybe(); + if (scudo::archSupportsMemoryTagging() && + !scudo::systemDetectsMemoryTagFaultsTestOnly()) + this->disableMemoryTagging(); } ~TestAllocator() { this->unmapTestOnly(); } }; @@ -180,8 +178,8 @@ bool Found = false; for (scudo::uptr I = 0; I < 1024U && !Found; I++) { void *P = Allocator->allocate(NeedleSize, Origin); - if (Allocator->untagPointerMaybe(P) == - Allocator->untagPointerMaybe(NeedleP)) + if (Allocator->getHeaderTaggedPointer(P) == + Allocator->getHeaderTaggedPointer(NeedleP)) Found = true; Allocator->deallocate(P, Origin); } @@ -248,38 +246,30 @@ Allocator->releaseToOS(); - if (Allocator->useMemoryTaggingTestOnly() && - scudo::systemDetectsMemoryTagFaultsTestOnly()) { - // Check that use-after-free is detected. - for (scudo::uptr SizeLog = 0U; SizeLog <= 20U; SizeLog++) { - const scudo::uptr Size = 1U << SizeLog; - if (!isTaggedAllocation(Allocator.get(), Size, 1)) - continue; - // UAF detection is probabilistic, so we repeat the test up to 256 times - // if necessary. With 15 possible tags this means a 1 in 15^256 chance of - // a false positive. - EXPECT_DEATH( - { - disableDebuggerdMaybe(); - for (unsigned I = 0; I != 256; ++I) { - void *P = Allocator->allocate(Size, Origin); - Allocator->deallocate(P, Origin); - reinterpret_cast(P)[0] = 0xaa; - } - }, - ""); - EXPECT_DEATH( - { - disableDebuggerdMaybe(); - for (unsigned I = 0; I != 256; ++I) { - void *P = Allocator->allocate(Size, Origin); - Allocator->deallocate(P, Origin); - reinterpret_cast(P)[Size - 1] = 0xaa; - } - }, - ""); - } + // Check that use-after-free is detected. + for (scudo::uptr SizeLog = 0U; SizeLog <= 20U; SizeLog++) { + const scudo::uptr Size = 1U << SizeLog; + if (!Allocator->useMemoryTaggingTestOnly()) + continue; + EXPECT_DEATH( + { + disableDebuggerdMaybe(); + void *P = Allocator->allocate(Size, Origin); + Allocator->deallocate(P, Origin); + reinterpret_cast(P)[0] = 0xaa; + }, + ""); + EXPECT_DEATH( + { + disableDebuggerdMaybe(); + void *P = Allocator->allocate(Size, Origin); + Allocator->deallocate(P, Origin); + reinterpret_cast(P)[Size - 1] = 0xaa; + }, + ""); + } + if (Allocator->useMemoryTaggingTestOnly()) { // Check that disabling memory tagging works correctly. void *P = Allocator->allocate(2048, Origin); EXPECT_DEATH(reinterpret_cast(P)[2048] = 0xaa, ""); @@ -289,7 +279,7 @@ Allocator->deallocate(P, Origin); P = Allocator->allocate(2048, Origin); - EXPECT_EQ(Allocator->untagPointerMaybe(P), P); + EXPECT_EQ(scudo::untagPointer(P), P); reinterpret_cast(P)[2048] = 0xaa; Allocator->deallocate(P, Origin); diff --git a/compiler-rt/lib/scudo/standalone/tests/secondary_test.cpp b/compiler-rt/lib/scudo/standalone/tests/secondary_test.cpp --- a/compiler-rt/lib/scudo/standalone/tests/secondary_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/secondary_test.cpp @@ -27,29 +27,29 @@ std::unique_ptr L(new SecondaryT); L->init(&S); const scudo::uptr Size = 1U << 16; - void *P = L->allocate(Size); + void *P = L->allocate(scudo::Options{}, Size); EXPECT_NE(P, nullptr); memset(P, 'A', Size); EXPECT_GE(SecondaryT::getBlockSize(P), Size); - L->deallocate(P); + L->deallocate(scudo::Options{}, P); // If the Secondary can't cache that pointer, it will be unmapped. if (!L->canCache(Size)) EXPECT_DEATH(memset(P, 'A', Size), ""); const scudo::uptr Align = 1U << 16; - P = L->allocate(Size + Align, Align); + P = L->allocate(scudo::Options{}, Size + Align, Align); EXPECT_NE(P, nullptr); void *AlignedP = reinterpret_cast( scudo::roundUpTo(reinterpret_cast(P), Align)); memset(AlignedP, 'A', Size); - L->deallocate(P); + L->deallocate(scudo::Options{}, P); std::vector V; for (scudo::uptr I = 0; I < 32U; I++) - V.push_back(L->allocate(Size)); + V.push_back(L->allocate(scudo::Options{}, Size)); std::shuffle(V.begin(), V.end(), std::mt19937(std::random_device()())); while (!V.empty()) { - L->deallocate(V.back()); + L->deallocate(scudo::Options{}, V.back()); V.pop_back(); } scudo::ScopedString Str(1024); @@ -59,11 +59,14 @@ struct NoCacheConfig { typedef scudo::MapAllocatorNoCache SecondaryCache; + static const bool MaySupportMemoryTagging = false; }; struct TestConfig { typedef scudo::MapAllocatorCache SecondaryCache; + static const bool MaySupportMemoryTagging = false; static const scudo::u32 SecondaryCacheEntriesArraySize = 128U; + static const scudo::u32 SecondaryCacheQuarantineSize = 0U; static const scudo::u32 SecondaryCacheDefaultMaxEntriesCount = 64U; static const scudo::uptr SecondaryCacheDefaultMaxEntrySize = 1UL << 20; static const scudo::s32 SecondaryCacheMinReleaseToOsIntervalMs = INT32_MIN; @@ -97,12 +100,12 @@ scudo::roundUpTo((1U << SizeLog) + Delta, MinAlign); const scudo::uptr Size = HeaderSize + UserSize + (Align > MinAlign ? Align - HeaderSize : 0); - void *P = L->allocate(Size, Align); + void *P = L->allocate(scudo::Options{}, Size, Align); EXPECT_NE(P, nullptr); void *AlignedP = reinterpret_cast( scudo::roundUpTo(reinterpret_cast(P), Align)); memset(AlignedP, 0xff, UserSize); - L->deallocate(P); + L->deallocate(scudo::Options{}, P); } } } @@ -117,7 +120,7 @@ std::vector V; const scudo::uptr PageSize = scudo::getPageSizeCached(); for (scudo::uptr I = 0; I < 32U; I++) - V.push_back(L->allocate((std::rand() % 16) * PageSize)); + V.push_back(L->allocate(scudo::Options{}, (std::rand() % 16) * PageSize)); auto Lambda = [V](scudo::uptr Block) { EXPECT_NE(std::find(V.begin(), V.end(), reinterpret_cast(Block)), V.end()); @@ -126,7 +129,7 @@ L->iterateOverBlocks(Lambda); L->enable(); while (!V.empty()) { - L->deallocate(V.back()); + L->deallocate(scudo::Options{}, V.back()); V.pop_back(); } scudo::ScopedString Str(1024); @@ -172,14 +175,14 @@ for (scudo::uptr I = 0; I < 128U; I++) { // Deallocate 75% of the blocks. const bool Deallocate = (rand() & 3) != 0; - void *P = L->allocate((std::rand() % 16) * PageSize); + void *P = L->allocate(scudo::Options{}, (std::rand() % 16) * PageSize); if (Deallocate) - L->deallocate(P); + L->deallocate(scudo::Options{}, P); else V.push_back(P); } while (!V.empty()) { - L->deallocate(V.back()); + L->deallocate(scudo::Options{}, V.back()); V.pop_back(); } }