[scudo][standalone] Optimization pass

[cryptoad]

Summary

This introduces a bunch of small optimizations with the purpose of
making the fastpath tighter:

• tag more conditions as LIKELY/UNLIKELY: as a rule of thumb we consider that every operation related to the secondary is unlikely
• attempt to reduce the number of potentially extraneous instructions
• reorganize the Chunk header to not straddle a word boundary and use more appropriate types

Note that some LIKELY/UNLIKELY impact might be less obvious as
they are in slow paths (for example in secondary.cc), but at this
point I am throwing a pretty wide net, and it's consistant and doesn't
hurt.

This was mosly done for the benfit of Android, but other platforms
benefit from it too. An aarch64 Android benchmark gives:

• before:

BM_youtube/min_time:15.000/repeats:4/manual_time_mean              445244 us       659385 us            4
BM_youtube/min_time:15.000/repeats:4/manual_time_median            445007 us       658970 us            4
BM_youtube/min_time:15.000/repeats:4/manual_time_stddev               885 us         1332 us            4

• after:

BM_youtube/min_time:15.000/repeats:4/manual_time_mean       415697 us       621925 us            4
BM_youtube/min_time:15.000/repeats:4/manual_time_median     415913 us       622061 us            4
BM_youtube/min_time:15.000/repeats:4/manual_time_stddev        990 us         1163 us            4

Additional since -Werror=conversion is enabled on some platforms we
are built on, enable it upstream to catch things early: a few sign
conversions had slept through and needed additional casting.

[cryptoad]

More optimizations:

BM_youtube/min_time:15.000/repeats:4/manual_time_mean       410274 us       615999 us            4
BM_youtube/min_time:15.000/repeats:4/manual_time_median     410246 us       616125 us            4
BM_youtube/min_time:15.000/repeats:4/manual_time_stddev        302 us          587 us            4

[vitalybuka]

Set of smaller patches may help you to bisect in case of regressions.

compiler-rt/trunk/lib/scudo/standalone/CMakeLists.txt

 add_compiler_rt_component(scudo_standalone)
 
 include_directories(../..)
 
 set(SCUDO_CFLAGS)
 
 list(APPEND SCUDO_CFLAGS
+  -Werror=conversion
   -Wall
   -nostdinc++)
 
 # Remove -stdlib= which is unused when passing -nostdinc++.
 string(REGEX REPLACE "-stdlib=[a-zA-Z+]*" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
 
 append_list_if(COMPILER_RT_HAS_FFREESTANDING_FLAG -ffreestanding SCUDO_CFLAGS)
 

compiler-rt/trunk/lib/scudo/standalone/chunk.h

INLINE u16 computeChecksum(u32 Seed, uptr Value, uptr *Array, uptr ArraySize) {
   // If the hardware CRC32 feature is defined here, it was enabled everywhere,
   // as opposed to only for crc32_hw.cc. This means that other hardware specific
   // instructions were likely emitted at other places, and as a result there is
   // no reason to not use it here.
 #if defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32)
   u32 Crc = static_cast<u32>(CRC32_INTRINSIC(Seed, Value));
   for (uptr I = 0; I < ArraySize; I++)
     Crc = static_cast<u32>(CRC32_INTRINSIC(Crc, Array[I]));
-  return static_cast<u16>((Crc & 0xffff) ^ (Crc >> 16));
+  return static_cast<u16>(Crc ^ (Crc >> 16));
 #else
   if (HashAlgorithm == Checksum::HardwareCRC32) {
     u32 Crc = computeHardwareCRC32(Seed, Value);
     for (uptr I = 0; I < ArraySize; I++)
       Crc = computeHardwareCRC32(Crc, Array[I]);
-    return static_cast<u16>((Crc & 0xffff) ^ (Crc >> 16));
+    return static_cast<u16>(Crc ^ (Crc >> 16));
   } else {
-    u16 Checksum = computeBSDChecksum(static_cast<u16>(Seed & 0xffff), Value);
+    u16 Checksum = computeBSDChecksum(static_cast<u16>(Seed), Value);
     for (uptr I = 0; I < ArraySize; I++)
       Checksum = computeBSDChecksum(Checksum, Array[I]);
     return Checksum;
   }
 #endif // defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32)
 }
 
 namespace Chunk {
enum Origin : u8 {
   Memalign = 3,
 };
 
 enum State : u8 { Available = 0, Allocated = 1, Quarantined = 2 };
 
 typedef u64 PackedHeader;
 // Update the 'Mask' constants to reflect changes in this structure.
 struct UnpackedHeader {
-  u64 Checksum : 16;
-  u64 ClassId : 8;
-  u64 SizeOrUnusedBytes : 20;
+  uptr ClassId : 8;
   u8 State : 2;
   u8 Origin : 2;
-  u64 Offset : 16;
+  uptr SizeOrUnusedBytes : 20;
+  uptr Offset : 16;
+  uptr Checksum : 16;
 };
 typedef atomic_u64 AtomicPackedHeader;
 COMPILER_CHECK(sizeof(UnpackedHeader) == sizeof(PackedHeader));
 
 // Those constants are required to silence some -Werror=conversion errors when
 // assigning values to the related bitfield variables.
-constexpr uptr ChecksumMask = (1UL << 16) - 1;
 constexpr uptr ClassIdMask = (1UL << 8) - 1;
+constexpr u8 StateMask = (1U << 2) - 1;
+constexpr u8 OriginMask = (1U << 2) - 1;
 constexpr uptr SizeOrUnusedBytesMask = (1UL << 20) - 1;
-constexpr uptr StateMask = (1UL << 2) - 1;
-constexpr uptr OriginMask = (1UL << 2) - 1;
 constexpr uptr OffsetMask = (1UL << 16) - 1;
+constexpr uptr ChecksumMask = (1UL << 16) - 1;
 
 constexpr uptr getHeaderSize() {
   return roundUpTo(sizeof(PackedHeader), 1U << SCUDO_MIN_ALIGNMENT_LOG);
 }
 
 INLINE AtomicPackedHeader *getAtomicHeader(void *Ptr) {
   return reinterpret_cast<AtomicPackedHeader *>(reinterpret_cast<uptr>(Ptr) -
                                                 getHeaderSize());

compiler-rt/trunk/lib/scudo/standalone/combined.h

void recycle(void *Ptr) {
       if (UNLIKELY(Header.State != Chunk::State::Quarantined))
         reportInvalidChunkState(AllocatorAction::Recycling, Ptr);
 
       Chunk::UnpackedHeader NewHeader = Header;
       NewHeader.State = Chunk::State::Available;
       Chunk::compareExchangeHeader(Allocator.Cookie, Ptr, &NewHeader, &Header);
 
       void *BlockBegin = Allocator::getBlockBegin(Ptr, &NewHeader);
-      const uptr ClassId = Header.ClassId;
-      if (ClassId)
+      const uptr ClassId = NewHeader.ClassId;
+      if (LIKELY(ClassId))
         Cache.deallocate(ClassId, BlockBegin);
       else
         Allocator.Secondary.deallocate(BlockBegin);
     }
 
     // We take a shortcut when allocating a quarantine batch by working with the
     // appropriate class ID instead of using Size. The compiler should optimize
     // the class ID computation and work with the associated cache directly.
void initLinkerInitialized() {
     initFlags();
     reportUnrecognizedFlags();
 
     // Store some flags locally.
     Options.MayReturnNull = getFlags()->may_return_null;
     Options.ZeroContents = getFlags()->zero_contents;
     Options.DeallocTypeMismatch = getFlags()->dealloc_type_mismatch;
     Options.DeleteSizeMismatch = getFlags()->delete_size_mismatch;
-    Options.QuarantineMaxChunkSize = getFlags()->quarantine_max_chunk_size;
+    Options.QuarantineMaxChunkSize =
+        static_cast<u32>(getFlags()->quarantine_max_chunk_size);
 
     Stats.initLinkerInitialized();
     Primary.initLinkerInitialized(getFlags()->release_to_os_interval_ms);
     Secondary.initLinkerInitialized(&Stats);
 
-    Quarantine.init(getFlags()->quarantine_size_kb << 10,
-                    getFlags()->thread_local_quarantine_size_kb << 10);
+    Quarantine.init(
+        static_cast<uptr>(getFlags()->quarantine_size_kb << 10),
+        static_cast<uptr>(getFlags()->thread_local_quarantine_size_kb << 10));
   }
 
   void reset() { memset(this, 0, sizeof(*this)); }
 
   void unmapTestOnly() {
     TSDRegistry.unmapTestOnly();
     Primary.unmapTestOnly();
   }
NOINLINE void *allocate(uptr Size, Chunk::Origin Origin,
                           bool ZeroContents = false) {
     initThreadMaybe();
 
     if (UNLIKELY(Alignment > MaxAlignment)) {
       if (Options.MayReturnNull)
         return nullptr;
       reportAlignmentTooBig(Alignment, MaxAlignment);
     }
-    if (UNLIKELY(Alignment < MinAlignment))
+    if (Alignment < MinAlignment)
       Alignment = MinAlignment;
 
     // If the requested size happens to be 0 (more common than you might think),
-    // allocate 1 byte on top of the header. Then add the extra bytes required
-    // to fulfill the alignment requirements: we allocate enough to be sure that
-    // there will be an address in the block that will satisfy the alignment.
+    // allocate MinAlignment bytes on top of the header. Then add the extra
+    // bytes required to fulfill the alignment requirements: we allocate enough
+    // to be sure that there will be an address in the block that will satisfy
+    // the alignment.
     const uptr NeededSize =
-        Chunk::getHeaderSize() + roundUpTo(Size ? Size : 1, MinAlignment) +
-        ((Alignment > MinAlignment) ? (Alignment - Chunk::getHeaderSize()) : 0);
+        roundUpTo(Size, MinAlignment) +
+        ((Alignment > MinAlignment) ? Alignment : Chunk::getHeaderSize());
 
     // Takes care of extravagantly large sizes as well as integer overflows.
     if (UNLIKELY(Size >= MaxAllowedMallocSize ||
                  NeededSize >= MaxAllowedMallocSize)) {
       if (Options.MayReturnNull)
         return nullptr;
       reportAllocationSizeTooBig(Size, NeededSize, MaxAllowedMallocSize);
     }
 
     void *Block;
     uptr ClassId;
-    uptr BlockEnd = 0;
-    if (PrimaryT::canAllocate(NeededSize)) {
+    uptr BlockEnd;
+    if (LIKELY(PrimaryT::canAllocate(NeededSize))) {
       ClassId = SizeClassMap::getClassIdBySize(NeededSize);
+      DCHECK_NE(ClassId, 0U);
       bool UnlockRequired;
       auto *TSD = TSDRegistry.getTSDAndLock(&UnlockRequired);
       Block = TSD->Cache.allocate(ClassId);
       if (UnlockRequired)
         TSD->unlock();
     } else {
       ClassId = 0;
       Block = Secondary.allocate(NeededSize, Alignment, &BlockEnd);
     }
 
     if (UNLIKELY(!Block)) {
       if (Options.MayReturnNull)
         return nullptr;
       reportOutOfMemory(NeededSize);
     }
 
-    // We only need to zero the contents for Primary backed allocations.
-    if ((ZeroContents || Options.ZeroContents) && ClassId)
+    // We only need to zero the contents for Primary backed allocations. This
+    // condition is not necessarily unlikely, but since memset is costly, we
+    // might as well mark it as such.
+    if (UNLIKELY((ZeroContents || Options.ZeroContents) && ClassId))
       memset(Block, 0, PrimaryT::getSizeByClassId(ClassId));
 
     Chunk::UnpackedHeader Header = {};
     uptr UserPtr = reinterpret_cast<uptr>(Block) + Chunk::getHeaderSize();
-    // The following condition isn't necessarily "UNLIKELY".
-    if (!isAligned(UserPtr, Alignment)) {
+    if (UNLIKELY(!isAligned(UserPtr, Alignment))) {
       const uptr AlignedUserPtr = roundUpTo(UserPtr, Alignment);
       const uptr Offset = AlignedUserPtr - UserPtr;
-      Header.Offset = (Offset >> MinAlignmentLog) & Chunk::OffsetMask;
       DCHECK_GT(Offset, 2 * sizeof(u32));
       // The BlockMarker has no security purpose, but is specifically meant for
       // the chunk iteration function that can be used in debugging situations.
       // It is the only situation where we have to locate the start of a chunk
       // based on its block address.
       reinterpret_cast<u32 *>(Block)[0] = BlockMarker;
       reinterpret_cast<u32 *>(Block)[1] = static_cast<u32>(Offset);
       UserPtr = AlignedUserPtr;
+      Header.Offset = (Offset >> MinAlignmentLog) & Chunk::OffsetMask;
     }
+    Header.ClassId = ClassId & Chunk::ClassIdMask;
     Header.State = Chunk::State::Allocated;
     Header.Origin = Origin & Chunk::OriginMask;
-    if (ClassId) {
-      Header.ClassId = ClassId & Chunk::ClassIdMask;
-      Header.SizeOrUnusedBytes = Size & Chunk::SizeOrUnusedBytesMask;
-    } else {
-      Header.SizeOrUnusedBytes =
-          (BlockEnd - (UserPtr + Size)) & Chunk::SizeOrUnusedBytesMask;
-    }
+    Header.SizeOrUnusedBytes = (ClassId ? Size : BlockEnd - (UserPtr + Size)) &
+                               Chunk::SizeOrUnusedBytesMask;
     void *Ptr = reinterpret_cast<void *>(UserPtr);
     Chunk::storeHeader(Cookie, Ptr, &Header);
 
     if (&__scudo_allocate_hook)
       __scudo_allocate_hook(Ptr, Size);
 
     return Ptr;
   }
if (Options.DeallocTypeMismatch) {
       if (UNLIKELY(OldHeader.Origin != Chunk::Origin::Malloc))
         reportDeallocTypeMismatch(AllocatorAction::Reallocating, OldPtr,
                                   OldHeader.Origin, Chunk::Origin::Malloc);
     }
 
     const uptr OldSize = getSize(OldPtr, &OldHeader);
     // If the new size is identical to the old one, or lower but within an
     // acceptable range, we just keep the old chunk, and update its header.
-    if (NewSize == OldSize)
+    if (UNLIKELY(NewSize == OldSize))
       return OldPtr;
     if (NewSize < OldSize) {
       const uptr Delta = OldSize - NewSize;
       if (Delta < (SizeClassMap::MaxSize / 2)) {
         Chunk::UnpackedHeader NewHeader = OldHeader;
         NewHeader.SizeOrUnusedBytes =
             (OldHeader.ClassId ? NewHeader.SizeOrUnusedBytes - Delta
                                : NewHeader.SizeOrUnusedBytes + Delta) &
if (UNLIKELY(Header.Offset != MaxOffset))
       reportSanityCheckError("offset");
 
     // Verify that we can fit the maximum size or amount of unused bytes in the
     // header. Given that the Secondary fits the allocation to a page, the worst
     // case scenario happens in the Primary. It will depend on the second to
     // last and last class sizes, as well as the dynamic base for the Primary.
     // The following is an over-approximation that works for our needs.
     const uptr MaxSizeOrUnusedBytes = SizeClassMap::MaxSize - 1;
-    Header.SizeOrUnusedBytes =
-        MaxSizeOrUnusedBytes & Chunk::SizeOrUnusedBytesMask;
+    Header.SizeOrUnusedBytes = MaxSizeOrUnusedBytes;
     if (UNLIKELY(Header.SizeOrUnusedBytes != MaxSizeOrUnusedBytes))
       reportSanityCheckError("size (or unused bytes)");
 
     const uptr LargestClassId = SizeClassMap::LargestClassId;
     Header.ClassId = LargestClassId;
     if (UNLIKELY(Header.ClassId != LargestClassId))
       reportSanityCheckError("class ID");
   }
 
   static INLINE void *getBlockBegin(const void *Ptr,
                                     Chunk::UnpackedHeader *Header) {
-    return reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) -
-                                    Chunk::getHeaderSize() -
-                                    (Header->Offset << MinAlignmentLog));
+    return reinterpret_cast<void *>(
+        reinterpret_cast<uptr>(Ptr) - Chunk::getHeaderSize() -
+        (static_cast<uptr>(Header->Offset) << MinAlignmentLog));
   }
 
   // Return the size of a chunk as requested during its allocation.
   INLINE uptr getSize(const void *Ptr, Chunk::UnpackedHeader *Header) {
     const uptr SizeOrUnusedBytes = Header->SizeOrUnusedBytes;
-    if (Header->ClassId)
+    if (LIKELY(Header->ClassId))
       return SizeOrUnusedBytes;
     return SecondaryT::getBlockEnd(getBlockBegin(Ptr, Header)) -
            reinterpret_cast<uptr>(Ptr) - SizeOrUnusedBytes;
   }
 
   ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) {
     TSDRegistry.initThreadMaybe(this, MinimalInit);
   }
 
   void quarantineOrDeallocateChunk(void *Ptr, Chunk::UnpackedHeader *Header,
                                    uptr Size) {
     Chunk::UnpackedHeader NewHeader = *Header;
     // If the quarantine is disabled, the actual size of a chunk is 0 or larger
     // than the maximum allowed, we return a chunk directly to the backend.
     const bool BypassQuarantine = !Quarantine.getCacheSize() || !Size ||
                                   (Size > Options.QuarantineMaxChunkSize);
     if (BypassQuarantine) {
       NewHeader.State = Chunk::State::Available;
       Chunk::compareExchangeHeader(Cookie, Ptr, &NewHeader, Header);
       void *BlockBegin = getBlockBegin(Ptr, &NewHeader);
       const uptr ClassId = NewHeader.ClassId;
-      if (ClassId) {
+      if (LIKELY(ClassId)) {
         bool UnlockRequired;
         auto *TSD = TSDRegistry.getTSDAndLock(&UnlockRequired);
         TSD->Cache.deallocate(ClassId, BlockBegin);
         if (UnlockRequired)
           TSD->unlock();
       } else {
         Secondary.deallocate(BlockBegin);
       }

compiler-rt/trunk/lib/scudo/standalone/fuchsia.cc

static void *allocateVmar(uptr Size, MapPlatformData *Data, bool AllowNoMem) {
   // Only scenario so far.
   DCHECK(Data);
   DCHECK_EQ(Data->Vmar, ZX_HANDLE_INVALID);
 
   const zx_status_t Status = _zx_vmar_allocate(
       _zx_vmar_root_self(),
       ZX_VM_CAN_MAP_READ | ZX_VM_CAN_MAP_WRITE | ZX_VM_CAN_MAP_SPECIFIC, 0,
       Size, &Data->Vmar, &Data->VmarBase);
-  if (Status != ZX_OK) {
+  if (UNLIKELY(Status != ZX_OK)) {
     if (Status != ZX_ERR_NO_MEMORY || !AllowNoMem)
       dieOnMapUnmapError(Status == ZX_ERR_NO_MEMORY);
     return nullptr;
   }
   return reinterpret_cast<void *>(Data->VmarBase);
 }
 
 void *map(void *Addr, uptr Size, const char *Name, uptr Flags,
if (Data && Data->Vmo != ZX_HANDLE_INVALID) {
     if (Status != ZX_OK) {
       if (Status != ZX_ERR_NO_MEMORY || !AllowNoMem)
         dieOnMapUnmapError(Status == ZX_ERR_NO_MEMORY);
       return nullptr;
     }
   } else {
     // Otherwise, create a Vmo and set its name.
     Status = _zx_vmo_create(Size, ZX_VMO_RESIZABLE, &Vmo);
-    if (Status != ZX_OK) {
+    if (UNLIKELY(Status != ZX_OK)) {
       if (Status != ZX_ERR_NO_MEMORY || !AllowNoMem)
         dieOnMapUnmapError(Status == ZX_ERR_NO_MEMORY);
       return nullptr;
     }
     _zx_object_set_property(Vmo, ZX_PROP_NAME, Name, strlen(Name));
   }
 
   uintptr_t P;
   zx_vm_option_t MapFlags =
       ZX_VM_PERM_READ | ZX_VM_PERM_WRITE | ZX_VM_ALLOW_FAULTS;
   const uint64_t Offset =
       Addr ? reinterpret_cast<uintptr_t>(Addr) - Data->VmarBase : 0;
   if (Offset)
     MapFlags |= ZX_VM_SPECIFIC;
   Status = _zx_vmar_map(Vmar, MapFlags, Offset, Vmo, VmoSize, Size, &P);
   // No need to track the Vmo if we don't intend on resizing it. Close it.
   if (Flags & MAP_RESIZABLE) {
     DCHECK(Data);
     DCHECK_EQ(Data->Vmo, ZX_HANDLE_INVALID);
     Data->Vmo = Vmo;
   } else {
     CHECK_EQ(_zx_handle_close(Vmo), ZX_OK);
   }
-  if (Status != ZX_OK) {
+  if (UNLIKELY(Status != ZX_OK)) {
     if (Status != ZX_ERR_NO_MEMORY || !AllowNoMem)
       dieOnMapUnmapError(Status == ZX_ERR_NO_MEMORY);
     return nullptr;
   }
   if (Data)
     Data->VmoSize += Size;
 
   return reinterpret_cast<void *>(P);
 }
 
 void unmap(void *Addr, uptr Size, uptr Flags, MapPlatformData *Data) {
   if (Flags & UNMAP_ALL) {
     DCHECK_NE(Data, nullptr);
     const zx_handle_t Vmar = Data->Vmar;
     DCHECK_NE(Vmar, _zx_vmar_root_self());
     // Destroying the vmar effectively unmaps the whole mapping.
     CHECK_EQ(_zx_vmar_destroy(Vmar), ZX_OK);
     CHECK_EQ(_zx_handle_close(Vmar), ZX_OK);
   } else {
     const zx_handle_t Vmar = Data ? Data->Vmar : _zx_vmar_root_self();
     const zx_status_t Status =
         _zx_vmar_unmap(Vmar, reinterpret_cast<uintptr_t>(Addr), Size);
-    if (Status != ZX_OK)
+    if (UNLIKELY(Status != ZX_OK))
       dieOnMapUnmapError();
   }
   if (Data) {
     if (Data->Vmo != ZX_HANDLE_INVALID)
       CHECK_EQ(_zx_handle_close(Data->Vmo), ZX_OK);
     memset(Data, 0, sizeof(*Data));
   }
 }

 }
 
 u64 getMonotonicTime() { return _zx_clock_get_monotonic(); }
 
 u32 getNumberOfCPUs() { return _zx_system_get_num_cpus(); }
 
 bool getRandom(void *Buffer, uptr Length, bool Blocking) {
   COMPILER_CHECK(MaxRandomLength <= ZX_CPRNG_DRAW_MAX_LEN);
-  if (!Buffer || !Length || Length > MaxRandomLength)
+  if (UNLIKELY(!Buffer || !Length || Length > MaxRandomLength))
     return false;
   _zx_cprng_draw(Buffer, Length);
   return true;
 }
 
 void outputRaw(const char *Buffer) {
   __sanitizer_log_write(Buffer, strlen(Buffer));
 }
 
 void setAbortMessage(const char *Message) {}
 
 } // namespace scudo
 
 #endif // SCUDO_FUCHSIA

compiler-rt/trunk/lib/scudo/standalone/local_cache.h

 
 template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
   typedef typename SizeClassAllocator::SizeClassMap SizeClassMap;
 
   struct TransferBatch {
     static const u32 MaxNumCached = SizeClassMap::MaxNumCachedHint;
     void setFromArray(void **Array, u32 N) {
       DCHECK_LE(N, MaxNumCached);
-      for (u32 I = 0; I < N; I++)
-        Batch[I] = Array[I];
       Count = N;
+      memcpy(Batch, Array, sizeof(void *) * Count);
     }
     void clear() { Count = 0; }
     void add(void *P) {
       DCHECK_LT(Count, MaxNumCached);
       Batch[Count++] = P;
     }
     void copyToArray(void **Array) const {
-      for (u32 I = 0; I < Count; I++)
-        Array[I] = Batch[I];
+      memcpy(Array, Batch, sizeof(void *) * Count);
     }
     u32 getCount() const { return Count; }
     void *get(u32 I) const {
       DCHECK_LE(I, Count);
       return Batch[I];
     }
     static u32 getMaxCached(uptr Size) {
       return Min(MaxNumCached, SizeClassMap::getMaxCachedHint(Size));
     }
     TransferBatch *Next;
 
   private:
     u32 Count;
     void *Batch[MaxNumCached];
   };
 
   void initLinkerInitialized(GlobalStats *S, SizeClassAllocator *A) {
     Stats.initLinkerInitialized();
-    if (S)
+    if (LIKELY(S))
       S->link(&Stats);
     Allocator = A;
   }
 
   void init(GlobalStats *S, SizeClassAllocator *A) {
     memset(this, 0, sizeof(*this));
     initLinkerInitialized(S, A);
   }
 
   void destroy(GlobalStats *S) {
     drain();
-    if (S)
+    if (LIKELY(S))
       S->unlink(&Stats);
   }
 
   void *allocate(uptr ClassId) {
-    CHECK_LT(ClassId, NumClasses);
+    DCHECK_LT(ClassId, NumClasses);
     PerClass *C = &PerClassArray[ClassId];
     if (C->Count == 0) {
       if (UNLIKELY(!refill(C, ClassId)))
         return nullptr;
       DCHECK_GT(C->Count, 0);
     }
     // We read ClassSize first before accessing Chunks because it's adjacent to
     // Count, while Chunks might be further off (depending on Count). That keeps
private:
   }
 
   NOINLINE bool refill(PerClass *C, uptr ClassId) {
     initCacheMaybe(C);
     TransferBatch *B = Allocator->popBatch(this, ClassId);
     if (UNLIKELY(!B))
       return false;
     DCHECK_GT(B->getCount(), 0);
-    B->copyToArray(C->Chunks);
     C->Count = B->getCount();
+    B->copyToArray(C->Chunks);
     destroyBatch(ClassId, B);
     return true;
   }
 
   NOINLINE void drain(PerClass *C, uptr ClassId) {
     const u32 Count = Min(C->MaxCount / 2, C->Count);
     const uptr FirstIndexToDrain = C->Count - Count;
     TransferBatch *B = createBatch(ClassId, C->Chunks[FirstIndexToDrain]);

compiler-rt/trunk/lib/scudo/standalone/mutex.h

 
 namespace scudo {
 
 class HybridMutex {
 public:
   void init() { memset(this, 0, sizeof(*this)); }
   bool tryLock();
   NOINLINE void lock() {
-    if (tryLock())
+    if (LIKELY(tryLock()))
       return;
-      // The compiler may try to fully unroll the loop, ending up in a
-      // NumberOfTries*NumberOfYields block of pauses mixed with tryLocks. This
-      // is large, ugly and unneeded, a compact loop is better for our purpose
-      // here. Use a pragma to tell the compiler not to unroll the loop.
 #ifdef __clang__
 #pragma nounroll
 #endif
     for (u8 I = 0U; I < NumberOfTries; I++) {
       yieldProcessor(NumberOfYields);
       if (tryLock())
         return;
     }
     lockSlow();
   }
   void unlock();
 
 private:
-  static constexpr u8 NumberOfTries = 10U;
-  static constexpr u8 NumberOfYields = 10U;
+  static constexpr u8 NumberOfTries = 8U;
+  static constexpr u8 NumberOfYields = 8U;
 
 #if SCUDO_LINUX
   atomic_u32 M;
 #elif SCUDO_FUCHSIA
   sync_mutex_t M;
 #endif
 
   void lockSlow();

compiler-rt/trunk/lib/scudo/standalone/primary32.h

if (UNLIKELY(!getRandom(reinterpret_cast<void *>(&Seed), sizeof(Seed))))
                            (reinterpret_cast<uptr>(SizeClassInfoArray) >> 6));
     const uptr PageSize = getPageSizeCached();
     for (uptr I = 0; I < NumClasses; I++) {
       SizeClassInfo *Sci = getSizeClassInfo(I);
       Sci->RandState = getRandomU32(&Seed);
       // See comment in the 64-bit primary about releasing smaller size classes.
       Sci->CanRelease = (ReleaseToOsInterval > 0) &&
                         (I != SizeClassMap::BatchClassId) &&
-                        (getSizeByClassId(I) >= (PageSize / 32));
+                        (getSizeByClassId(I) >= (PageSize / 16));
     }
     ReleaseToOsIntervalMs = ReleaseToOsInterval;
   }
   void init(s32 ReleaseToOsInterval) {
     memset(this, 0, sizeof(*this));
     initLinkerInitialized(ReleaseToOsInterval);
   }
 
   void unmapTestOnly() {
     while (NumberOfStashedRegions > 0)
       unmap(reinterpret_cast<void *>(RegionsStash[--NumberOfStashedRegions]),
             RegionSize);
     // TODO(kostyak): unmap the TransferBatch regions as well.
     for (uptr I = 0; I < NumRegions; I++)
       if (PossibleRegions[I])
         unmap(reinterpret_cast<void *>(I * RegionSize), RegionSize);
     PossibleRegions.unmapTestOnly();
   }
 
   TransferBatch *popBatch(CacheT *C, uptr ClassId) {
     DCHECK_LT(ClassId, NumClasses);
     SizeClassInfo *Sci = getSizeClassInfo(ClassId);
     ScopedLock L(Sci->Mutex);
     TransferBatch *B = Sci->FreeList.front();
-    if (B)
+    if (B) {
       Sci->FreeList.pop_front();
-    else {
+    } else {
       B = populateFreeList(C, ClassId, Sci);
       if (UNLIKELY(!B))
         return nullptr;
     }
     DCHECK_GT(B->getCount(), 0);
     Sci->Stats.PoppedBlocks += B->getCount();
     return B;
   }
public:
 
   void disable() {
     for (uptr I = 0; I < NumClasses; I++)
       getSizeClassInfo(I)->Mutex.lock();
   }
 
   void enable() {
     for (sptr I = static_cast<sptr>(NumClasses) - 1; I >= 0; I--)
-      getSizeClassInfo(I)->Mutex.unlock();
+      getSizeClassInfo(static_cast<uptr>(I))->Mutex.unlock();
   }
 
   template <typename F> void iterateOverBlocks(F Callback) {
     for (uptr I = MinRegionIndex; I <= MaxRegionIndex; I++)
       if (PossibleRegions[I]) {
         const uptr BlockSize = getSizeByClassId(PossibleRegions[I]);
         const uptr From = I * RegionSize;
         const uptr To = From + (RegionSize / BlockSize) * BlockSize;
if ((Sci->Stats.PushedBlocks - Sci->ReleaseInfo.PushedBlocksAtLastRelease) *
         PageSize) {
       return; // Nothing new to release.
     }
 
     if (!Force) {
       const s32 IntervalMs = ReleaseToOsIntervalMs;
       if (IntervalMs < 0)
         return;
-      if (Sci->ReleaseInfo.LastReleaseAtNs + IntervalMs * 1000000ULL >
+      if (Sci->ReleaseInfo.LastReleaseAtNs +
+              static_cast<uptr>(IntervalMs) * 1000000ULL >
           getMonotonicTime()) {
         return; // Memory was returned recently.
       }
     }
 
     // TODO(kostyak): currently not ideal as we loop over all regions and
     // iterate multiple times over the same freelist if a ClassId spans multiple
     // regions. But it will have to do for now.

compiler-rt/trunk/lib/scudo/standalone/primary64.h

for (uptr I = 0; I < NumClasses; I++) {
       // meaningful RSS impact: there are more blocks per page, they are
       // randomized around, and thus pages are less likely to be entirely empty.
       // On top of this, attempting to release those require more iterations and
       // memory accesses which ends up being fairly costly. The current lower
       // limit is mostly arbitrary and based on empirical observations.
       // TODO(kostyak): make the lower limit a runtime option
       Region->CanRelease = (ReleaseToOsInterval > 0) &&
                            (I != SizeClassMap::BatchClassId) &&
-                           (getSizeByClassId(I) >= (PageSize / 32));
+                           (getSizeByClassId(I) >= (PageSize / 16));
       Region->RandState = getRandomU32(&Seed);
     }
     ReleaseToOsIntervalMs = ReleaseToOsInterval;
   }
   void init(s32 ReleaseToOsInterval) {
     memset(this, 0, sizeof(*this));
     initLinkerInitialized(ReleaseToOsInterval);
   }
 
   void unmapTestOnly() {
     unmap(reinterpret_cast<void *>(PrimaryBase), PrimarySize, UNMAP_ALL, &Data);
     unmap(reinterpret_cast<void *>(RegionInfoArray),
           sizeof(RegionInfo) * NumClasses);
   }
 
   TransferBatch *popBatch(CacheT *C, uptr ClassId) {
     DCHECK_LT(ClassId, NumClasses);
     RegionInfo *Region = getRegionInfo(ClassId);
     ScopedLock L(Region->Mutex);
     TransferBatch *B = Region->FreeList.front();
-    if (B)
+    if (B) {
       Region->FreeList.pop_front();
-    else {
+    } else {
       B = populateFreeList(C, ClassId, Region);
       if (UNLIKELY(!B))
         return nullptr;
     }
     DCHECK_GT(B->getCount(), 0);
     Region->Stats.PoppedBlocks += B->getCount();
     return B;
   }
public:
 
   void disable() {
     for (uptr I = 0; I < NumClasses; I++)
       getRegionInfo(I)->Mutex.lock();
   }
 
   void enable() {
     for (sptr I = static_cast<sptr>(NumClasses) - 1; I >= 0; I--)
-      getRegionInfo(I)->Mutex.unlock();
+      getRegionInfo(static_cast<uptr>(I))->Mutex.unlock();
   }
 
   template <typename F> void iterateOverBlocks(F Callback) const {
     for (uptr I = 1; I < NumClasses; I++) {
       const RegionInfo *Region = getRegionInfo(I);
       const uptr BlockSize = getSizeByClassId(I);
       const uptr From = Region->RegionBeg;
       const uptr To = From + Region->AllocatedUser;
public:
   }
 
 private:
   static const uptr RegionSize = 1UL << RegionSizeLog;
   static const uptr NumClasses = SizeClassMap::NumClasses;
   static const uptr PrimarySize = RegionSize * NumClasses;
 
   // Call map for user memory with at least this size.
-  static const uptr MapSizeIncrement = 1UL << 16;
+  static const uptr MapSizeIncrement = 1UL << 17;
 
   struct RegionStats {
     uptr PoppedBlocks;
     uptr PushedBlocks;
   };
 
   struct ReleaseToOsInfo {
     uptr PushedBlocksAtLastRelease;
if (LIKELY(TotalUserBytes > MappedUser)) {
           Region->Exhausted = true;
           printStats();
           Printf(
               "Scudo OOM: The process has Exhausted %zuM for size class %zu.\n",
               RegionSize >> 20, Size);
         }
         return nullptr;
       }
-      if (MappedUser == 0)
+      if (UNLIKELY(MappedUser == 0))
         Region->Data = Data;
       if (UNLIKELY(!map(reinterpret_cast<void *>(RegionBeg + MappedUser),
                         UserMapSize, "scudo:primary",
                         MAP_ALLOWNOMEM | MAP_RESIZABLE, &Region->Data)))
         return nullptr;
       Region->MappedUser += UserMapSize;
       C->getStats().add(StatMapped, UserMapSize);
     }
for (uptr I = P; I < P + AllocatedUser; I += Size) {
       }
     }
     if (Count) {
       if (UNLIKELY(!populateBatches(C, Region, ClassId, &B, MaxCount,
                                     ShuffleArray, Count)))
         return nullptr;
     }
     DCHECK(B);
-    CHECK_GT(B->getCount(), 0);
+    DCHECK_GT(B->getCount(), 0);
 
     Region->AllocatedUser += AllocatedUser;
     Region->Exhausted = false;
     if (Region->CanRelease)
       Region->ReleaseInfo.LastReleaseAtNs = getMonotonicTime();
 
     return B;
   }
if ((Region->Stats.PushedBlocks -
         PageSize) {
       return; // Nothing new to release.
     }
 
     if (!Force) {
       const s32 IntervalMs = ReleaseToOsIntervalMs;
       if (IntervalMs < 0)
         return;
-      if (Region->ReleaseInfo.LastReleaseAtNs + IntervalMs * 1000000ULL >
+      if (Region->ReleaseInfo.LastReleaseAtNs +
+              static_cast<uptr>(IntervalMs) * 1000000ULL >
           getMonotonicTime()) {
         return; // Memory was returned recently.
       }
     }
 
     ReleaseRecorder Recorder(Region->RegionBeg, &Region->Data);
     releaseFreeMemoryToOS(&Region->FreeList, Region->RegionBeg,
                           roundUpTo(Region->AllocatedUser, PageSize) / PageSize,

compiler-rt/trunk/lib/scudo/standalone/secondary.h

 }
 
 } // namespace LargeBlock
 
 class MapAllocator {
 public:
   void initLinkerInitialized(GlobalStats *S) {
     Stats.initLinkerInitialized();
-    if (S)
+    if (LIKELY(S))
       S->link(&Stats);
   }
   void init(GlobalStats *S) {
     memset(this, 0, sizeof(*this));
     initLinkerInitialized(S);
   }
 
   void *allocate(uptr Size, uptr AlignmentHint = 0, uptr *BlockEnd = nullptr);

compiler-rt/trunk/lib/scudo/standalone/secondary.cc

void *MapAllocator::allocate(uptr Size, uptr AlignmentHint, uptr *BlockEnd) {
   DCHECK_GT(Size, AlignmentHint);
   const uptr PageSize = getPageSizeCached();
   const uptr MapSize =
       roundUpTo(Size + LargeBlock::getHeaderSize(), PageSize) + 2 * PageSize;
   MapPlatformData Data = {};
   uptr MapBase =
       reinterpret_cast<uptr>(map(nullptr, MapSize, "scudo:secondary",
                                  MAP_NOACCESS | MAP_ALLOWNOMEM, &Data));
-  if (!MapBase)
+  if (UNLIKELY(!MapBase))
     return nullptr;
   uptr CommitBase = MapBase + PageSize;
   uptr MapEnd = MapBase + MapSize;
 
   // In the unlikely event of alignments larger than a page, adjust the amount
   // of memory we want to commit, and trim the extra memory.
-  if (AlignmentHint >= PageSize) {
+  if (UNLIKELY(AlignmentHint >= PageSize)) {
     // For alignments greater than or equal to a page, the user pointer (eg: the
     // pointer that is returned by the C or C++ allocation APIs) ends up on a
     // page boundary , and our headers will live in the preceding page.
     CommitBase = roundUpTo(MapBase + PageSize + 1, AlignmentHint) - PageSize;
     const uptr NewMapBase = CommitBase - PageSize;
     DCHECK_GE(NewMapBase, MapBase);
     // We only trim the extra memory on 32-bit platforms: 64-bit platforms
     // are less constrained memory wise, and that saves us two syscalls.
const uptr Ptr =
                                  CommitSize, "scudo:secondary", 0, &Data));
   LargeBlock::Header *H = reinterpret_cast<LargeBlock::Header *>(Ptr);
   H->MapBase = MapBase;
   H->MapSize = MapEnd - MapBase;
   H->BlockEnd = CommitBase + CommitSize;
   H->Data = Data;
   {
     ScopedLock L(Mutex);
-    if (!Tail) {
-      Tail = H;
-    } else {
+    if (LIKELY(Tail)) {
       Tail->Next = H;
       H->Prev = Tail;
-      Tail = H;
     }
+    Tail = H;
     AllocatedBytes += CommitSize;
     if (LargestSize < CommitSize)
       LargestSize = CommitSize;
     NumberOfAllocs++;
     Stats.add(StatAllocated, CommitSize);
     Stats.add(StatMapped, H->MapSize);
   }
   if (BlockEnd)
LargeBlock::Header *H = LargeBlock::getHeader(Ptr);
     if (Prev) {
       CHECK_EQ(Prev->Next, H);
       Prev->Next = Next;
     }
     if (Next) {
       CHECK_EQ(Next->Prev, H);
       Next->Prev = Prev;
     }
-    if (Tail == H) {
+    if (UNLIKELY(Tail == H)) {
       CHECK(!Next);
       Tail = Prev;
     } else {
       CHECK(Next);
     }
     const uptr CommitSize = H->BlockEnd - reinterpret_cast<uptr>(H);
     FreedBytes += CommitSize;
     NumberOfFrees++;

compiler-rt/trunk/lib/scudo/standalone/size_class_map.h

public:
   }
 };
 
 typedef SizeClassMap<3, 5, 8, 17, 8, 10> DefaultSizeClassMap;
 
 // TODO(kostyak): further tune class maps for Android & Fuchsia.
 #if SCUDO_WORDSIZE == 64U
 typedef SizeClassMap<3, 5, 8, 15, 8, 10> SvelteSizeClassMap;
-typedef SizeClassMap<3, 5, 8, 16, 14, 12> AndroidSizeClassMap;
+typedef SizeClassMap<3, 5, 8, 17, 14, 14> AndroidSizeClassMap;
 #else
 typedef SizeClassMap<3, 4, 7, 15, 8, 10> SvelteSizeClassMap;
-typedef SizeClassMap<3, 4, 7, 16, 14, 12> AndroidSizeClassMap;
+typedef SizeClassMap<3, 4, 7, 17, 14, 14> AndroidSizeClassMap;
 #endif
 
 } // namespace scudo
 
 #endif // SCUDO_SIZE_CLASS_MAP_H_

compiler-rt/trunk/lib/scudo/standalone/string_utils.cc

 //===-- string_utils.cc -----------------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "string_utils.h"
 #include "common.h"
 
-#include <ctype.h>
 #include <stdarg.h>
 #include <string.h>
 
 namespace scudo {
 
 static int appendChar(char **Buffer, const char *BufferEnd, char C) {
   if (*Buffer < BufferEnd) {
     **Buffer = C;
if (Negative && MinNumberLength)
     --MinNumberLength;
   if (Negative && PadWithZero)
     Res += appendChar(Buffer, BufferEnd, '-');
   uptr NumBuffer[MaxLen];
   int Pos = 0;
   do {
     RAW_CHECK_MSG(static_cast<uptr>(Pos) < MaxLen,
                   "appendNumber buffer overflow");
-    NumBuffer[Pos++] = AbsoluteValue % Base;
+    NumBuffer[Pos++] = static_cast<uptr>(AbsoluteValue % Base);
     AbsoluteValue /= Base;
   } while (AbsoluteValue > 0);
   if (Pos < MinNumberLength) {
     memset(&NumBuffer[Pos], 0,
            sizeof(NumBuffer[0]) * static_cast<uptr>(MinNumberLength - Pos));
     Pos = MinNumberLength;
   }
   RAW_CHECK(Pos > 0);
static int appendPointer(char **Buffer, const char *BufferEnd, u64 ptr_value) {
   Res += appendUnsigned(Buffer, BufferEnd, ptr_value, 16,
                         SCUDO_POINTER_FORMAT_LENGTH, /*PadWithZero=*/true,
                         /*Upper=*/false);
   return Res;
 }
 
 int formatString(char *Buffer, uptr BufferLength, const char *Format,
                  va_list Args) {
-  UNUSED static const char *PrintfFormatsHelp =
+  static const char *PrintfFormatsHelp =
       "Supported formatString formats: %([0-9]*)?(z|ll)?{d,u,x,X}; %p; "
       "%[-]([0-9]*)?(\\.\\*)?s; %c\n";
   RAW_CHECK(Format);
   RAW_CHECK(BufferLength > 0);
   const char *BufferEnd = &Buffer[BufferLength - 1];
   const char *Cur = Format;
   int Res = 0;
   for (; *Cur; Cur++) {

compiler-rt/trunk/lib/scudo/standalone/tsd_exclusive.h

ALWAYS_INLINE TSD<Allocator> *getTSDAndLock(bool *UnlockRequired) {
     FallbackTSD->lock();
     *UnlockRequired = true;
     return FallbackTSD;
   }
 
 private:
   void initOnceMaybe(Allocator *Instance) {
     ScopedLock L(Mutex);
-    if (Initialized)
+    if (LIKELY(Initialized))
       return;
     initLinkerInitialized(Instance); // Sets Initialized.
   }
 
   // Using minimal initialization allows for global initialization while keeping
   // the thread specific structure untouched. The fallback structure will be
   // used instead.
   NOINLINE void initThread(Allocator *Instance, bool MinimalInit) {
     initOnceMaybe(Instance);
-    if (MinimalInit)
+    if (UNLIKELY(MinimalInit))
       return;
     CHECK_EQ(
         pthread_setspecific(PThreadKey, reinterpret_cast<void *>(Instance)), 0);
     ThreadTSD.initLinkerInitialized(Instance);
     State = ThreadState::Initialized;
   }
 
   pthread_key_t PThreadKey;

compiler-rt/trunk/lib/scudo/standalone/tsd_shared.h

#elif SCUDO_LINUX
     return ThreadTSD;
 #else
     return reinterpret_cast<TSD<Allocator> *>(pthread_getspecific(PThreadKey));
 #endif
   }
 
   void initOnceMaybe(Allocator *Instance) {
     ScopedLock L(Mutex);
-    if (Initialized)
+    if (LIKELY(Initialized))
       return;
     initLinkerInitialized(Instance); // Sets Initialized.
   }
 
   NOINLINE void initThread(Allocator *Instance) {
     initOnceMaybe(Instance);
     // Initial context assignment is done in a plain round-robin fashion.
     const u32 Index = atomic_fetch_add(&CurrentIndex, 1U, memory_order_relaxed);
     setCurrentTSD(&TSDs[Index % NumberOfTSDs]);
   }
 
   NOINLINE TSD<Allocator> *getTSDAndLockSlow(TSD<Allocator> *CurrentTSD) {
     if (MaxTSDCount > 1U && NumberOfTSDs > 1U) {
       // Use the Precedence of the current TSD as our random seed. Since we are
       // in the slow path, it means that tryLock failed, and as a result it's
       // very likely that said Precedence is non-zero.
-      u32 RandState = static_cast<u32>(CurrentTSD->getPrecedence());
-      const u32 R = getRandomU32(&RandState);
+      const u32 R = static_cast<u32>(CurrentTSD->getPrecedence());
       const u32 Inc = CoPrimes[R % NumberOfCoPrimes];
       u32 Index = R % NumberOfTSDs;
       uptr LowestPrecedence = UINTPTR_MAX;
       TSD<Allocator> *CandidateTSD = nullptr;
       // Go randomly through at most 4 contexts and find a candidate.
       for (u32 I = 0; I < Min(4U, NumberOfTSDs); I++) {
         if (TSDs[Index].tryLock()) {
           setCurrentTSD(&TSDs[Index]);