Cache size per class size in SizeClassAllocatorXLocalCache.

[alekseyshl]

Summary

Allocator::ClassIdToSize() is not free and calling it in every
Allocate/Deallocate has noticeable impact on perf.

[eugenis]

LGTM

compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator_local_cache.h

void Destroy(SizeClassAllocator *allocator, AllocatorGlobalStats *s) {
     Drain(allocator);
     if (s)
       s->Unregister(&stats_);
   }
 
   void *Allocate(SizeClassAllocator *allocator, uptr class_id) {
     CHECK_NE(class_id, 0UL);
     CHECK_LT(class_id, kNumClasses);
-    stats_.Add(AllocatorStatAllocated, Allocator::ClassIdToSize(class_id));
     PerClass *c = &per_class_[class_id];
+    stats_.Add(AllocatorStatAllocated, c->class_size);
     if (UNLIKELY(c->count == 0))
       Refill(c, allocator, class_id);
     CHECK_GT(c->count, 0);
     CompactPtrT chunk = c->chunks[--c->count];
     void *res = reinterpret_cast<void *>(allocator->CompactPtrToPointer(
         allocator->GetRegionBeginBySizeClass(class_id), chunk));
     return res;
   }
 
   void Deallocate(SizeClassAllocator *allocator, uptr class_id, void *p) {
     CHECK_NE(class_id, 0UL);
     CHECK_LT(class_id, kNumClasses);
     // If the first allocator call on a new thread is a deallocation, then
     // max_count will be zero, leading to check failure.
     InitCache();
-    stats_.Sub(AllocatorStatAllocated, Allocator::ClassIdToSize(class_id));
     PerClass *c = &per_class_[class_id];
+    stats_.Sub(AllocatorStatAllocated, c->class_size);
     CHECK_NE(c->max_count, 0UL);
     if (UNLIKELY(c->count == c->max_count))
       Drain(c, allocator, class_id, c->max_count / 2);
     CompactPtrT chunk = allocator->PointerToCompactPtr(
         allocator->GetRegionBeginBySizeClass(class_id),
         reinterpret_cast<uptr>(p));
     c->chunks[c->count++] = chunk;
   }
 
   void Drain(SizeClassAllocator *allocator) {
     for (uptr class_id = 0; class_id < kNumClasses; class_id++) {
       PerClass *c = &per_class_[class_id];
       while (c->count > 0)
         Drain(c, allocator, class_id, c->count);
     }
   }
 
   // private:
   struct PerClass {
     u32 count;
     u32 max_count;
+    uptr class_size;
     CompactPtrT chunks[2 * SizeClassMap::kMaxNumCachedHint];
   };
   PerClass per_class_[kNumClasses];
   AllocatorStats stats_;
 
   void InitCache() {
     if (per_class_[1].max_count)
       return;
     for (uptr i = 0; i < kNumClasses; i++) {
       PerClass *c = &per_class_[i];
       c->max_count = 2 * SizeClassMap::MaxCachedHint(i);
+      c->class_size = Allocator::ClassIdToSize(i);
     }
   }
 
   NOINLINE void Refill(PerClass *c, SizeClassAllocator *allocator,
                        uptr class_id) {
     InitCache();
     uptr num_requested_chunks = c->max_count / 2;
     allocator->GetFromAllocator(&stats_, class_id, c->chunks,
void Destroy(SizeClassAllocator *allocator, AllocatorGlobalStats *s) {
     Drain(allocator);
     if (s)
       s->Unregister(&stats_);
   }
 
   void *Allocate(SizeClassAllocator *allocator, uptr class_id) {
     CHECK_NE(class_id, 0UL);
     CHECK_LT(class_id, kNumClasses);
-    stats_.Add(AllocatorStatAllocated, Allocator::ClassIdToSize(class_id));
     PerClass *c = &per_class_[class_id];
+    stats_.Add(AllocatorStatAllocated, c->class_size);
     if (UNLIKELY(c->count == 0))
       Refill(allocator, class_id);
     void *res = c->batch[--c->count];
     PREFETCH(c->batch[c->count - 1]);
     return res;
   }
 
   void Deallocate(SizeClassAllocator *allocator, uptr class_id, void *p) {
     CHECK_NE(class_id, 0UL);
     CHECK_LT(class_id, kNumClasses);
     // If the first allocator call on a new thread is a deallocation, then
     // max_count will be zero, leading to check failure.
     InitCache();
-    stats_.Sub(AllocatorStatAllocated, Allocator::ClassIdToSize(class_id));
     PerClass *c = &per_class_[class_id];
+    stats_.Sub(AllocatorStatAllocated, c->class_size);
     CHECK_NE(c->max_count, 0UL);
     if (UNLIKELY(c->count == c->max_count))
       Drain(allocator, class_id);
     c->batch[c->count++] = p;
   }
 
   void Drain(SizeClassAllocator *allocator) {
     for (uptr class_id = 0; class_id < kNumClasses; class_id++) {
       PerClass *c = &per_class_[class_id];
       while (c->count > 0)
         Drain(allocator, class_id);
     }
   }
 
   // private:
   typedef typename SizeClassAllocator::SizeClassMapT SizeClassMap;
   struct PerClass {
     uptr count;
     uptr max_count;
+    uptr class_size;
     void *batch[2 * TransferBatch::kMaxNumCached];
   };
   PerClass per_class_[kNumClasses];
   AllocatorStats stats_;
 
   void InitCache() {
     if (per_class_[1].max_count)
       return;
     for (uptr i = 0; i < kNumClasses; i++) {
       PerClass *c = &per_class_[i];
       c->max_count = 2 * TransferBatch::MaxCached(i);
+      c->class_size = Allocator::ClassIdToSize(i);
     }
   }
 
   // TransferBatch class is declared in SizeClassAllocator.
   // We transfer chunks between central and thread-local free lists in batches.
   // For small size classes we allocate batches separately.
   // For large size classes we may use one of the chunks to store the batch.
   // sizeof(TransferBatch) must be a power of 2 for more efficient allocation.