diff --git a/compiler-rt/lib/scudo/standalone/mem_map.h b/compiler-rt/lib/scudo/standalone/mem_map.h --- a/compiler-rt/lib/scudo/standalone/mem_map.h +++ b/compiler-rt/lib/scudo/standalone/mem_map.h @@ -34,6 +34,7 @@ bool mapImpl(uptr Addr, uptr Size, const char *Name, uptr Flags); void unmapImpl(uptr Addr, uptr Size); bool remapImpl(uptr Addr, uptr Size, const char *Name, uptr Flags); + void setMemoryPermissionImpl(uptr Addr, uptr Size, uptr Flags); void releasePagesToOSImpl(uptr From, uptr Size) { return releaseAndZeroPagesToOSImpl(From, Size); } diff --git a/compiler-rt/lib/scudo/standalone/mem_map.cpp b/compiler-rt/lib/scudo/standalone/mem_map.cpp --- a/compiler-rt/lib/scudo/standalone/mem_map.cpp +++ b/compiler-rt/lib/scudo/standalone/mem_map.cpp @@ -46,6 +46,10 @@ return ::scudo::releasePagesToOS(Base, From - Base, Size, &Data); } +void MemMapDefault::setMemoryPermissionImpl(uptr Addr, uptr Size, uptr Flags) { + return ::scudo::setMemoryPermission(Addr, Size, Flags); +} + void ReservedMemoryDefault::releaseImpl() { ::scudo::unmap(reinterpret_cast(Base), Capacity, UNMAP_ALL, &Data); } diff --git a/compiler-rt/lib/scudo/standalone/mem_map_base.h b/compiler-rt/lib/scudo/standalone/mem_map_base.h --- a/compiler-rt/lib/scudo/standalone/mem_map_base.h +++ b/compiler-rt/lib/scudo/standalone/mem_map_base.h @@ -45,6 +45,15 @@ return invokeImpl(&Derived::remapImpl, Addr, Size, Name, Flags); } + // This is used to update the pages' access permission. For example, mark + // pages as no read/write permission. + void setMemoryPermission(uptr Addr, uptr Size, uptr Flags) { + DCHECK(isAllocated()); + DCHECK((Addr >= getBase()) || (Addr + Size <= getBase() + getCapacity())); + return static_cast(this)->setMemoryPermissionImpl(Addr, Size, + Flags); + } + // Suggest releasing a set of contiguous physical pages back to the OS. Note // that only physical pages are supposed to be released. Any release of // virtual pages may lead to undefined behavior. diff --git a/compiler-rt/lib/scudo/standalone/secondary.h b/compiler-rt/lib/scudo/standalone/secondary.h --- a/compiler-rt/lib/scudo/standalone/secondary.h +++ b/compiler-rt/lib/scudo/standalone/secondary.h @@ -12,6 +12,7 @@ #include "chunk.h" #include "common.h" #include "list.h" +#include "mem_map.h" #include "memtag.h" #include "mutex.h" #include "options.h" @@ -37,9 +38,7 @@ LargeBlock::Header *Next; uptr CommitBase; uptr CommitSize; - uptr MapBase; - uptr MapSize; - [[no_unique_address]] MapPlatformData Data; + MemMapT MemMap; }; static_assert(sizeof(Header) % (1U << SCUDO_MIN_ALIGNMENT_LOG) == 0, ""); @@ -66,8 +65,11 @@ } // namespace LargeBlock static void unmap(LargeBlock::Header *H) { - MapPlatformData Data = H->Data; - unmap(reinterpret_cast(H->MapBase), H->MapSize, UNMAP_ALL, &Data); + // Note that the `H->MapMap` is stored on the pages managed by itself. Take + // over the ownership before unmap() so that any operation along with unmap() + // won't touch inaccessible pages. + MemMapT MemMap = H->MemMap; + MemMap.unmap(MemMap.getBase(), MemMap.getCapacity()); } class MapAllocatorNoCache { @@ -97,20 +99,19 @@ template void mapSecondary(Options Options, uptr CommitBase, uptr CommitSize, - uptr AllocPos, uptr Flags, MapPlatformData *Data) { + uptr AllocPos, uptr Flags, MemMapT &MemMap) { const uptr MaxUnusedCacheBytes = MaxUnusedCachePages * getPageSizeCached(); if (useMemoryTagging(Options) && CommitSize > MaxUnusedCacheBytes) { const uptr UntaggedPos = Max(AllocPos, CommitBase + MaxUnusedCacheBytes); - map(reinterpret_cast(CommitBase), UntaggedPos - CommitBase, - "scudo:secondary", MAP_RESIZABLE | MAP_MEMTAG | Flags, Data); - map(reinterpret_cast(UntaggedPos), - CommitBase + CommitSize - UntaggedPos, "scudo:secondary", - MAP_RESIZABLE | Flags, Data); + MemMap.remap(CommitBase, UntaggedPos - CommitBase, "scudo:secondary", + MAP_RESIZABLE | MAP_MEMTAG | Flags); + MemMap.remap(UntaggedPos, CommitBase + CommitSize - UntaggedPos, + "scudo:secondary", MAP_RESIZABLE | Flags); } else { - map(reinterpret_cast(CommitBase), CommitSize, "scudo:secondary", + const uptr RemapFlags = MAP_RESIZABLE | (useMemoryTagging(Options) ? MAP_MEMTAG : 0) | - Flags, - Data); + Flags; + MemMap.remap(CommitBase, CommitSize, "scudo:secondary", RemapFlags); } } @@ -155,10 +156,8 @@ CachedBlock Entry; Entry.CommitBase = H->CommitBase; Entry.CommitSize = H->CommitSize; - Entry.MapBase = H->MapBase; - Entry.MapSize = H->MapSize; Entry.BlockBegin = reinterpret_cast(H + 1); - Entry.Data = H->Data; + Entry.MemMap = H->MemMap; Entry.Time = Time; if (useMemoryTagging(Options)) { if (Interval == 0 && !SCUDO_FUCHSIA) { @@ -168,13 +167,13 @@ // on top so we just do the two syscalls there. Entry.Time = 0; mapSecondary(Options, Entry.CommitBase, Entry.CommitSize, - Entry.CommitBase, MAP_NOACCESS, &Entry.Data); + Entry.CommitBase, MAP_NOACCESS, Entry.MemMap); } else { - setMemoryPermission(Entry.CommitBase, Entry.CommitSize, MAP_NOACCESS, - &Entry.Data); + Entry.MemMap.setMemoryPermission(Entry.CommitBase, Entry.CommitSize, + MAP_NOACCESS); } } else if (Interval == 0) { - releasePagesToOS(Entry.CommitBase, 0, Entry.CommitSize, &Entry.Data); + Entry.MemMap.releasePagesToOS(Entry.CommitBase, Entry.CommitSize); Entry.Time = 0; } do { @@ -223,8 +222,7 @@ else if (Interval >= 0) releaseOlderThan(Time - static_cast(Interval) * 1000000); if (!EntryCached) - unmap(reinterpret_cast(Entry.MapBase), Entry.MapSize, UNMAP_ALL, - &Entry.Data); + Entry.MemMap.unmap(Entry.MemMap.getBase(), Entry.MemMap.getCapacity()); } bool retrieve(Options Options, uptr Size, uptr Alignment, @@ -267,7 +265,7 @@ LargeBlock::addHeaderTag(HeaderPos)); *Zeroed = Entry.Time == 0; if (useMemoryTagging(Options)) - setMemoryPermission(Entry.CommitBase, Entry.CommitSize, 0, &Entry.Data); + Entry.MemMap.setMemoryPermission(Entry.CommitBase, Entry.CommitSize, 0); uptr NewBlockBegin = reinterpret_cast(*H + 1); if (useMemoryTagging(Options)) { if (*Zeroed) { @@ -282,9 +280,7 @@ } (*H)->CommitBase = Entry.CommitBase; (*H)->CommitSize = Entry.CommitSize; - (*H)->MapBase = Entry.MapBase; - (*H)->MapSize = Entry.MapSize; - (*H)->Data = Entry.Data; + (*H)->MemMap = Entry.MemMap; return true; } @@ -323,16 +319,18 @@ ScopedLock L(Mutex); for (u32 I = 0; I != Config::SecondaryCacheQuarantineSize; ++I) { if (Quarantine[I].CommitBase) { - unmap(reinterpret_cast(Quarantine[I].MapBase), - Quarantine[I].MapSize, UNMAP_ALL, &Quarantine[I].Data); + MemMapT &MemMap = Quarantine[I].MemMap; + MemMap.unmap(MemMap.getBase(), MemMap.getCapacity()); Quarantine[I].CommitBase = 0; } } const u32 MaxCount = atomic_load_relaxed(&MaxEntriesCount); - for (u32 I = 0; I < MaxCount; I++) - if (Entries[I].CommitBase) - setMemoryPermission(Entries[I].CommitBase, Entries[I].CommitSize, 0, - &Entries[I].Data); + for (u32 I = 0; I < MaxCount; I++) { + if (Entries[I].CommitBase) { + Entries[I].MemMap.setMemoryPermission(Entries[I].CommitBase, + Entries[I].CommitSize, 0); + } + } QuarantinePos = -1U; } @@ -345,9 +343,7 @@ private: void empty() { struct { - void *MapBase; - uptr MapSize; - MapPlatformData Data; + MemMapT MemMap; } MapInfo[Config::SecondaryCacheEntriesArraySize]; uptr N = 0; { @@ -355,27 +351,24 @@ for (uptr I = 0; I < Config::SecondaryCacheEntriesArraySize; I++) { if (!Entries[I].CommitBase) continue; - MapInfo[N].MapBase = reinterpret_cast(Entries[I].MapBase); - MapInfo[N].MapSize = Entries[I].MapSize; - MapInfo[N].Data = Entries[I].Data; + MapInfo[N].MemMap = Entries[I].MemMap; Entries[I].CommitBase = 0; N++; } EntriesCount = 0; IsFullEvents = 0; } - for (uptr I = 0; I < N; I++) - unmap(MapInfo[I].MapBase, MapInfo[I].MapSize, UNMAP_ALL, - &MapInfo[I].Data); + for (uptr I = 0; I < N; I++) { + MemMapT &MemMap = MapInfo[I].MemMap; + MemMap.unmap(MemMap.getBase(), MemMap.getCapacity()); + } } struct CachedBlock { uptr CommitBase; uptr CommitSize; - uptr MapBase; - uptr MapSize; uptr BlockBegin; - [[no_unique_address]] MapPlatformData Data; + MemMapT MemMap; u64 Time; }; @@ -387,7 +380,7 @@ OldestTime = Entry.Time; return; } - releasePagesToOS(Entry.CommitBase, 0, Entry.CommitSize, &Entry.Data); + Entry.MemMap.releasePagesToOS(Entry.CommitBase, Entry.CommitSize); Entry.Time = 0; } @@ -538,16 +531,20 @@ AllocatedBytes += BlockSize; NumberOfAllocs++; Stats.add(StatAllocated, BlockSize); - Stats.add(StatMapped, H->MapSize); + Stats.add(StatMapped, H->MemMap.getCapacity()); } return Ptr; } } - MapPlatformData Data = {}; + ReservedMemoryT ReservedMemory; const uptr MapSize = RoundedSize + 2 * PageSize; - uptr MapBase = reinterpret_cast( - map(nullptr, MapSize, nullptr, MAP_NOACCESS | MAP_ALLOWNOMEM, &Data)); + ReservedMemory.create(/*Addr=*/0U, MapSize, nullptr, MAP_ALLOWNOMEM); + + // Take the entire ownership of reserved region. + MemMapT MemMap = ReservedMemory.dispatch(ReservedMemory.getBase(), + ReservedMemory.getCapacity()); + uptr MapBase = MemMap.getBase(); if (UNLIKELY(!MapBase)) return nullptr; uptr CommitBase = MapBase + PageSize; @@ -565,21 +562,21 @@ // We only trim the extra memory on 32-bit platforms: 64-bit platforms // are less constrained memory wise, and that saves us two syscalls. if (SCUDO_WORDSIZE == 32U && NewMapBase != MapBase) { - unmap(reinterpret_cast(MapBase), NewMapBase - MapBase, 0, &Data); + MemMap.unmap(MapBase, NewMapBase - MapBase); MapBase = NewMapBase; } const uptr NewMapEnd = CommitBase + PageSize + roundUp(Size, PageSize) + PageSize; DCHECK_LE(NewMapEnd, MapEnd); if (SCUDO_WORDSIZE == 32U && NewMapEnd != MapEnd) { - unmap(reinterpret_cast(NewMapEnd), MapEnd - NewMapEnd, 0, &Data); + MemMap.unmap(NewMapEnd, MapEnd - NewMapEnd); MapEnd = NewMapEnd; } } const uptr CommitSize = MapEnd - PageSize - CommitBase; const uptr AllocPos = roundDown(CommitBase + CommitSize - Size, Alignment); - mapSecondary(Options, CommitBase, CommitSize, AllocPos, 0, &Data); + mapSecondary(Options, CommitBase, CommitSize, AllocPos, 0, MemMap); const uptr HeaderPos = AllocPos - Chunk::getHeaderSize() - LargeBlock::getHeaderSize(); LargeBlock::Header *H = reinterpret_cast( @@ -587,11 +584,9 @@ if (useMemoryTagging(Options)) storeTags(LargeBlock::addHeaderTag(CommitBase), reinterpret_cast(H + 1)); - H->MapBase = MapBase; - H->MapSize = MapEnd - MapBase; H->CommitBase = CommitBase; H->CommitSize = CommitSize; - H->Data = Data; + H->MemMap = MemMap; if (BlockEndPtr) *BlockEndPtr = CommitBase + CommitSize; { @@ -602,7 +597,7 @@ LargestSize = CommitSize; NumberOfAllocs++; Stats.add(StatAllocated, CommitSize); - Stats.add(StatMapped, H->MapSize); + Stats.add(StatMapped, H->MemMap.getCapacity()); } return reinterpret_cast(HeaderPos + LargeBlock::getHeaderSize()); } @@ -618,7 +613,7 @@ FreedBytes += CommitSize; NumberOfFrees++; Stats.sub(StatAllocated, CommitSize); - Stats.sub(StatMapped, H->MapSize); + Stats.sub(StatMapped, H->MemMap.getCapacity()); } Cache.store(Options, H); }