diff --git a/compiler-rt/lib/scudo/standalone/common.h b/compiler-rt/lib/scudo/standalone/common.h --- a/compiler-rt/lib/scudo/standalone/common.h +++ b/compiler-rt/lib/scudo/standalone/common.h @@ -168,8 +168,8 @@ void setMemoryPermission(uptr Addr, uptr Size, uptr Flags, MapPlatformData *Data = nullptr); -void releasePagesToOS(uptr BaseAddress, uptr Offset, uptr Size, - MapPlatformData *Data = nullptr); +void releasePagesToOS(uptr BaseAddress, uptr Offset, uptr Size, uptr Flags, + MapPlatformData *Data); // Internal map & unmap fatal error. This must not call map(). void NORETURN dieOnMapUnmapError(bool OutOfMemory = false); diff --git a/compiler-rt/lib/scudo/standalone/fuchsia.cpp b/compiler-rt/lib/scudo/standalone/fuchsia.cpp --- a/compiler-rt/lib/scudo/standalone/fuchsia.cpp +++ b/compiler-rt/lib/scudo/standalone/fuchsia.cpp @@ -147,7 +147,7 @@ } void releasePagesToOS(UNUSED uptr BaseAddress, uptr Offset, uptr Size, - MapPlatformData *Data) { + UNUSED uptr Flags, MapPlatformData *Data) { DCHECK(Data); DCHECK_NE(Data->Vmar, ZX_HANDLE_INVALID); DCHECK_NE(Data->Vmo, ZX_HANDLE_INVALID); diff --git a/compiler-rt/lib/scudo/standalone/linux.cpp b/compiler-rt/lib/scudo/standalone/linux.cpp --- a/compiler-rt/lib/scudo/standalone/linux.cpp +++ b/compiler-rt/lib/scudo/standalone/linux.cpp @@ -36,6 +36,12 @@ #define ANDROID_PR_SET_VMA_ANON_NAME 0 #endif +#if defined(__aarch64__) +#ifndef PROT_MTE +#define PROT_MTE 0x20 +#endif +#endif + namespace scudo { uptr getPageSize() { return static_cast(sysconf(_SC_PAGESIZE)); } @@ -53,9 +59,6 @@ MmapProt = PROT_READ | PROT_WRITE; } #if defined(__aarch64__) -#ifndef PROT_MTE -#define PROT_MTE 0x20 -#endif if (Flags & MAP_MEMTAG) MmapProt |= PROT_MTE; #endif @@ -116,14 +119,24 @@ return NeedsMemset == Yes; } -void releasePagesToOS(uptr BaseAddress, uptr Offset, uptr Size, +void releasePagesToOS(uptr BaseAddress, uptr Offset, uptr Size, uptr Flags, UNUSED MapPlatformData *Data) { void *Addr = reinterpret_cast(BaseAddress + Offset); if (madviseNeedsMemsetCached()) { // Workaround for QEMU-user ignoring MADV_DONTNEED. // https://github.com/qemu/qemu/blob/b1cffefa1b163bce9aebc3416f562c1d3886eeaa/linux-user/syscall.c#L11941 // https://bugs.launchpad.net/qemu/+bug/1926521 - memset(Addr, 0, Size); + int Prot = (Flags & MAP_NOACCESS) ? PROT_NONE : (PROT_READ | PROT_WRITE); +#if defined(__aarch64__) + if (Flags & MAP_MEMTAG) + Prot |= PROT_MTE; +#endif + void *P = + mmap(Addr, Size, Prot, + MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); + if (P != Addr) + dieOnMapUnmapError(); + return; } while (madvise(Addr, Size, MADV_DONTNEED) == -1 && errno == EAGAIN) { } diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h --- a/compiler-rt/lib/scudo/standalone/primary32.h +++ b/compiler-rt/lib/scudo/standalone/primary32.h @@ -240,6 +240,7 @@ static const uptr NumRegions = SCUDO_MMAP_RANGE_SIZE >> Config::PrimaryRegionSizeLog; static const u32 MaxNumBatches = SCUDO_ANDROID ? 4U : 8U; + static const uptr MapFlags = MAP_ALLOWNOMEM; typedef FlatByteMap ByteMap; struct SizeClassStats { @@ -279,7 +280,7 @@ uptr allocateRegionSlow() { uptr MapSize = 2 * RegionSize; const uptr MapBase = reinterpret_cast( - map(nullptr, MapSize, "scudo:primary", MAP_ALLOWNOMEM)); + map(nullptr, MapSize, "scudo:primary", MapFlags)); if (!MapBase) return 0; const uptr MapEnd = MapBase + MapSize; @@ -469,7 +470,7 @@ uptr TotalReleasedBytes = 0; const uptr Base = First * RegionSize; const uptr NumberOfRegions = Last - First + 1U; - ReleaseRecorder Recorder(Base); + ReleaseRecorder Recorder(Base, MapFlags, nullptr); auto SkipRegion = [this, First, ClassId](uptr RegionIndex) { return (PossibleRegions[First + RegionIndex] - 1U) != ClassId; }; diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h --- a/compiler-rt/lib/scudo/standalone/primary64.h +++ b/compiler-rt/lib/scudo/standalone/primary64.h @@ -264,6 +264,10 @@ static const uptr RegionSize = 1UL << Config::PrimaryRegionSizeLog; static const uptr NumClasses = SizeClassMap::NumClasses; static const uptr PrimarySize = RegionSize * NumClasses; + uptr GetMapFlags() const { + return MAP_ALLOWNOMEM | MAP_RESIZABLE | + (useMemoryTagging(Options.load()) ? MAP_MEMTAG : 0); + } // Call map for user memory with at least this size. static const uptr MapSizeIncrement = 1UL << 18; @@ -350,12 +354,9 @@ } if (MappedUser == 0) Region->Data = Data; - if (UNLIKELY(!map( - reinterpret_cast(RegionBeg + MappedUser), MapSize, - "scudo:primary", - MAP_ALLOWNOMEM | MAP_RESIZABLE | - (useMemoryTagging(Options.load()) ? MAP_MEMTAG : 0), - &Region->Data))) + if (UNLIKELY(!map(reinterpret_cast(RegionBeg + MappedUser), + MapSize, "scudo:primary", GetMapFlags(), + &Region->Data))) return nullptr; Region->MappedUser += MapSize; C->getStats().add(StatMapped, MapSize); @@ -458,7 +459,7 @@ } } - ReleaseRecorder Recorder(Region->RegionBeg, &Region->Data); + ReleaseRecorder Recorder(Region->RegionBeg, GetMapFlags(), &Region->Data); const uptr CompactPtrBase = getCompactPtrBaseByClassId(ClassId); auto DecompactPtr = [CompactPtrBase](CompactPtrT CompactPtr) { return decompactPtrInternal(CompactPtrBase, CompactPtr); diff --git a/compiler-rt/lib/scudo/standalone/release.h b/compiler-rt/lib/scudo/standalone/release.h --- a/compiler-rt/lib/scudo/standalone/release.h +++ b/compiler-rt/lib/scudo/standalone/release.h @@ -17,8 +17,8 @@ class ReleaseRecorder { public: - ReleaseRecorder(uptr Base, MapPlatformData *Data = nullptr) - : Base(Base), Data(Data) {} + ReleaseRecorder(uptr Base, uptr Flags, MapPlatformData *Data) + : Base(Base), Flags(Flags), Data(Data) {} uptr getReleasedRangesCount() const { return ReleasedRangesCount; } @@ -29,7 +29,7 @@ // Releases [From, To) range of pages back to OS. void releasePageRangeToOS(uptr From, uptr To) { const uptr Size = To - From; - releasePagesToOS(Base, From, Size, Data); + releasePagesToOS(Base, From, Size, Flags, Data); ReleasedRangesCount++; ReleasedBytes += Size; } @@ -38,6 +38,7 @@ uptr ReleasedRangesCount = 0; uptr ReleasedBytes = 0; uptr Base = 0; + uptr Flags = MAP_NOACCESS; MapPlatformData *Data = nullptr; }; diff --git a/compiler-rt/lib/scudo/standalone/secondary.h b/compiler-rt/lib/scudo/standalone/secondary.h --- a/compiler-rt/lib/scudo/standalone/secondary.h +++ b/compiler-rt/lib/scudo/standalone/secondary.h @@ -38,6 +38,7 @@ uptr CommitSize; uptr MapBase; uptr MapSize; + uptr MapFlags; [[no_unique_address]] MapPlatformData Data; }; @@ -146,10 +147,12 @@ Entry.CommitSize = H->CommitSize; Entry.MapBase = H->MapBase; Entry.MapSize = H->MapSize; + Entry.MapFlags = H->MapFlags; Entry.BlockBegin = reinterpret_cast(H + 1); Entry.Data = H->Data; Entry.Time = Time; if (useMemoryTagging(Options)) { + Entry.MapFlags = MAP_NOACCESS; if (Interval == 0 && !SCUDO_FUCHSIA) { // Release the memory and make it inaccessible at the same time by // creating a new MAP_NOACCESS mapping on top of the existing mapping. @@ -157,13 +160,14 @@ // on top so we just do the two syscalls there. Entry.Time = 0; mapSecondary(Options, Entry.CommitBase, Entry.CommitSize, - Entry.CommitBase, MAP_NOACCESS, &Entry.Data); + Entry.CommitBase, Entry.MapFlags, &Entry.Data); } else { - setMemoryPermission(Entry.CommitBase, Entry.CommitSize, MAP_NOACCESS, + setMemoryPermission(Entry.CommitBase, Entry.CommitSize, Entry.MapFlags, &Entry.Data); } } else if (Interval == 0) { - releasePagesToOS(Entry.CommitBase, 0, Entry.CommitSize, &Entry.Data); + releasePagesToOS(Entry.CommitBase, 0, Entry.CommitSize, Entry.MapFlags, + &Entry.Data); Entry.Time = 0; } do { @@ -251,8 +255,11 @@ *H = reinterpret_cast( LargeBlock::addHeaderTag(HeaderPos)); *Zeroed = Entry.Time == 0; - if (useMemoryTagging(Options)) - setMemoryPermission(Entry.CommitBase, Entry.CommitSize, 0, &Entry.Data); + if (useMemoryTagging(Options)) { + Entry.MapFlags = 0; + setMemoryPermission(Entry.CommitBase, Entry.CommitSize, Entry.MapFlags, + &Entry.Data); + } uptr NewBlockBegin = reinterpret_cast(*H + 1); if (useMemoryTagging(Options)) { if (*Zeroed) @@ -269,6 +276,7 @@ (*H)->MapBase = Entry.MapBase; (*H)->MapSize = Entry.MapSize; (*H)->Data = Entry.Data; + (*H)->MapFlags = Entry.MapFlags; EntriesCount--; } return Found; @@ -316,9 +324,11 @@ } const u32 MaxCount = atomic_load_relaxed(&MaxEntriesCount); for (u32 I = 0; I < MaxCount; I++) - if (Entries[I].CommitBase) - setMemoryPermission(Entries[I].CommitBase, Entries[I].CommitSize, 0, - &Entries[I].Data); + if (Entries[I].CommitBase) { + Entries[I].MapFlags = 0; + setMemoryPermission(Entries[I].CommitBase, Entries[I].CommitSize, + Entries[I].MapFlags, &Entries[I].Data); + } QuarantinePos = -1U; } @@ -359,6 +369,7 @@ uptr MapBase; uptr MapSize; uptr BlockBegin; + uptr MapFlags; [[no_unique_address]] MapPlatformData Data; u64 Time; }; @@ -371,7 +382,8 @@ OldestTime = Entry.Time; return; } - releasePagesToOS(Entry.CommitBase, 0, Entry.CommitSize, &Entry.Data); + releasePagesToOS(Entry.CommitBase, 0, Entry.CommitSize, Entry.MapFlags, + &Entry.Data); Entry.Time = 0; } @@ -558,7 +570,9 @@ const uptr CommitSize = MapEnd - PageSize - CommitBase; const uptr AllocPos = roundDownTo(CommitBase + CommitSize - Size, Alignment); - mapSecondary(Options, CommitBase, CommitSize, AllocPos, 0, &Data); + const uptr MapFlags = 0; + mapSecondary(Options, CommitBase, CommitSize, AllocPos, MapFlags, + &Data); const uptr HeaderPos = AllocPos - Chunk::getHeaderSize() - LargeBlock::getHeaderSize(); LargeBlock::Header *H = reinterpret_cast( @@ -571,6 +585,7 @@ H->CommitBase = CommitBase; H->CommitSize = CommitSize; H->Data = Data; + H->MapFlags = MapFlags; if (BlockEndPtr) *BlockEndPtr = CommitBase + CommitSize; { diff --git a/compiler-rt/lib/scudo/standalone/tests/common_test.cpp b/compiler-rt/lib/scudo/standalone/tests/common_test.cpp --- a/compiler-rt/lib/scudo/standalone/tests/common_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/common_test.cpp @@ -46,10 +46,9 @@ EXPECT_EQ(std::count(P, P + N, 0), 0); EXPECT_LT(getResidentMemorySize() - Size, Threshold); - releasePagesToOS((uptr)P, 0, Size, &Data); + releasePagesToOS((uptr)P, 0, Size, 0, &Data); EXPECT_EQ(std::count(P, P + N, 0), N); - // FIXME: does not work with QEMU-user. - // EXPECT_LT(getResidentMemorySize() - OnStart, Threshold); + EXPECT_LT(getResidentMemorySize() - OnStart, Threshold); memset(P, 1, Size); EXPECT_EQ(std::count(P, P + N, 0), 0);