diff --git a/compiler-rt/lib/scudo/standalone/secondary.h b/compiler-rt/lib/scudo/standalone/secondary.h --- a/compiler-rt/lib/scudo/standalone/secondary.h +++ b/compiler-rt/lib/scudo/standalone/secondary.h @@ -89,8 +89,11 @@ template class MapAllocatorNoCache { public: void init(UNUSED s32 ReleaseToOsInterval) {} - bool retrieve(UNUSED uptr Size, UNUSED CachedBlock &Entry) { return false; } - void store(UNUSED const Options &Options, LargeBlock::Header *H) { unmap(H); } + bool retrieve(UNUSED Options Options, UNUSED uptr Size, UNUSED uptr Alignment, + UNUSED LargeBlock::Header **H, UNUSED bool *Zeroed) { + return false; + } + void store(UNUSED Options Options, LargeBlock::Header *H) { unmap(H); } bool canCache(UNUSED uptr Size) { return false; } void disable() {} void enable() {} @@ -259,9 +262,13 @@ Entry.MemMap.unmap(Entry.MemMap.getBase(), Entry.MemMap.getCapacity()); } - bool retrieve(uptr Size, CachedBlock &Entry) EXCLUDES(Mutex) { + bool retrieve(Options Options, uptr Size, uptr Alignment, + LargeBlock::Header **H, bool *Zeroed) EXCLUDES(Mutex) { + const uptr PageSize = getPageSizeCached(); const u32 MaxCount = atomic_load_relaxed(&MaxEntriesCount); bool Found = false; + CachedBlock Entry; + uptr HeaderPos = 0; { ScopedLock L(Mutex); if (EntriesCount == 0) @@ -269,8 +276,18 @@ for (u32 I = 0; I < MaxCount; I++) { if (!Entries[I].isValid()) continue; - if (Size > Entries[I].CommitSize) + const uptr CommitBase = Entries[I].CommitBase; + const uptr CommitSize = Entries[I].CommitSize; + const uptr AllocPos = + roundDown(CommitBase + CommitSize - Size, Alignment); + HeaderPos = + AllocPos - Chunk::getHeaderSize() - LargeBlock::getHeaderSize(); + if (HeaderPos > CommitBase + CommitSize) continue; + if (HeaderPos < CommitBase || + AllocPos > CommitBase + PageSize * MaxUnusedCachePages) { + continue; + } Found = true; Entry = Entries[I]; Entries[I].invalidate(); @@ -278,7 +295,29 @@ break; } } - return Found; + if (!Found) + return false; + + *H = reinterpret_cast( + LargeBlock::addHeaderTag(HeaderPos)); + *Zeroed = Entry.Time == 0; + if (useMemoryTagging(Options)) + Entry.MemMap.setMemoryPermission(Entry.CommitBase, Entry.CommitSize, 0); + uptr NewBlockBegin = reinterpret_cast(*H + 1); + if (useMemoryTagging(Options)) { + if (*Zeroed) { + storeTags(LargeBlock::addHeaderTag(Entry.CommitBase), + NewBlockBegin); + } else if (Entry.BlockBegin < NewBlockBegin) { + storeTags(Entry.BlockBegin, NewBlockBegin); + } else { + storeTags(untagPointer(NewBlockBegin), untagPointer(Entry.BlockBegin)); + } + } + (*H)->CommitBase = Entry.CommitBase; + (*H)->CommitSize = Entry.CommitSize; + (*H)->MemMap = Entry.MemMap; + return true; } bool canCache(uptr Size) { @@ -443,27 +482,6 @@ } } - inline void setHeader(const Options &Options, CachedBlock &Entry, - LargeBlock::Header *H, bool &Zeroed) { - Zeroed = Entry.Time == 0; - if (useMemoryTagging(Options)) { - Entry.MemMap.setMemoryPermission(Entry.CommitBase, Entry.CommitSize, 0); - // Block begins after the LargeBlock::Header - uptr NewBlockBegin = reinterpret_cast(H + 1); - if (Zeroed) { - storeTags(LargeBlock::addHeaderTag(Entry.CommitBase), - NewBlockBegin); - } else if (Entry.BlockBegin < NewBlockBegin) { - storeTags(Entry.BlockBegin, NewBlockBegin); - } else { - storeTags(untagPointer(NewBlockBegin), untagPointer(Entry.BlockBegin)); - } - } - H->CommitBase = Entry.CommitBase; - H->CommitSize = Entry.CommitSize; - H->MemMap = Entry.MemMap; - } - bool canCache(uptr Size) { return Cache.canCache(Size); } bool setOption(Option O, sptr Value) { return Cache.setOption(O, Value); } @@ -518,15 +536,7 @@ if (Alignment < PageSize && Cache.canCache(RoundedSize)) { LargeBlock::Header *H; bool Zeroed; - CachedBlock Entry; - if (Cache.retrieve(RoundedSize, Entry)) { - const uptr AllocPos = - roundDown(Entry.CommitBase + Entry.CommitSize - Size, Alignment); - const uptr HeaderPos = - AllocPos - LargeBlock::getHeaderSize() - Chunk::getHeaderSize(); - H = reinterpret_cast( - LargeBlock::addHeaderTag(HeaderPos)); - setHeader(Options, Entry, H, Zeroed); + if (Cache.retrieve(Options, Size, Alignment, &H, &Zeroed)) { const uptr BlockEnd = H->CommitBase + H->CommitSize; if (BlockEndPtr) *BlockEndPtr = BlockEnd;