[memprof] Move the meminfo block struct to MemProfData.inc.

[snehasish]

Summary

The definition of the MemInfoBlock is shared between the memprof
compiler-rt runtime and llvm/lib/ProfileData/. This change removes the
memprof_meminfoblock header and moves the struct to the shared include
file. To enable this sharing, the Print method is moved to the
memprof_allocator (the only place it is used) and the remaining uses are
updated to refer to the MemInfoBlock defined in the MemProfData.inc
file.

Also a couple of other minor changes which improve usability of the
types in MemProfData.inc.

• Update the PACKED macro to handle commas.
• Add constructors and equality operators.
• Don't initialize the buildid field.

[davidxl]

compiler-rt/lib/memprof/tests/rawprofile.cpp

136

More explanation on this change?

[snehasish]

Address comments.

[snehasish]

compiler-rt/lib/memprof/tests/rawprofile.cpp

136

Duplicated the comment I had in the .inc file. It's useful to record the linker build id (i.e. the uuid field here: https://git.io/J9O47) however, this is not implemented in sanitizer_procmaps_linux yet. Our initial approach was to do symbolization using internal tooling and this was necessary. However, now we are going to symbolize using llvm-profdata and this is not strictly necessary anymore. It is still good to have for sanity checks as well as other visualization and analysis tools that we may build in the future.

[davidxl]

lgtm

compiler-rt/include/profile/MemProfData.inc

  * This file has two identical copies. The primary copy lives in LLVM and
  * the other one sits in compiler-rt/include/profile directory. To make changes
  * in this file, first modify the primary copy and copy it over to compiler-rt.
  * Testing of any change in this file can start only after the two copies are
  * synced up.
  *
 \*===----------------------------------------------------------------------===*/
 
-
 #ifdef _MSC_VER
-#define PACKED(__decl__) __pragma(pack(push,1)) __decl__ __pragma(pack(pop))
+#define PACKED(...) __pragma(pack(push,1)) __VA_ARGS__ __pragma(pack(pop))
 #else
-#define PACKED(__decl__) __decl__ __attribute__((__packed__))
+#define PACKED(...) __VA_ARGS__ __attribute__((__packed__))
 #endif
 
 // A 64-bit magic number to uniquely identify the raw binary memprof profile file.
 #define MEMPROF_RAW_MAGIC_64                                                                        \
   ((uint64_t)255 << 56 | (uint64_t)'m' << 48 | (uint64_t)'p' << 40 | (uint64_t)'r' << 32 |          \
    (uint64_t)'o' << 24 | (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129)
 
 // The version number of the raw binary format.
 #define MEMPROF_RAW_VERSION 1ULL
 
+#include <string.h>
+
 namespace llvm {
 namespace memprof {
 // A struct describing the header used for the raw binary memprof profile format.
 PACKED(struct Header {
   uint64_t Magic;
   uint64_t Version;
   uint64_t TotalSize;
   uint64_t SegmentOffset;
   uint64_t MIBOffset;
   uint64_t StackOffset;
 });
 
+
 // A struct describing the information necessary to describe a /proc/maps
 // segment entry for a particular binary/library identified by its build id.
 PACKED(struct SegmentEntry {
   uint64_t Start;
   uint64_t End;
   uint64_t Offset;
-  uint8_t BuildId[32];
+  // This field is unused until sanitizer procmaps support for build ids for
+  // Linux Elf is implemented.
+  uint8_t BuildId[32] = {0};
+
+  SegmentEntry(uint64_t S, uint64_t E, uint64_t O) :
+    Start(S), End(E), Offset(O) {}
+
+  SegmentEntry(const SegmentEntry& S) {
+    Start = S.Start;
+    End = S.End;
+    Offset = S.Offset;
+  }
+
+  SegmentEntry& operator=(const SegmentEntry& S) {
+    Start = S.Start;
+    End = S.End;
+    Offset = S.Offset;
+    return *this;
+  }
+
+  bool operator==(const SegmentEntry& S) const {
+    return Start == S.Start &&
+           End == S.End &&
+           Offset == S.Offset;
+  }
 });
+
+// A struct representing the heap allocation characteristics of a particular
+// runtime context. This struct is shared between the compiler-rt runtime and
+// the raw profile reader. The indexed format uses a separate, self-describing
+// backwards compatible format.
+PACKED(struct MemInfoBlock {
+  uint32_t alloc_count;
+  uint64_t total_access_count, min_access_count, max_access_count;
+  uint64_t total_size;
+  uint32_t min_size, max_size;
+  uint32_t alloc_timestamp, dealloc_timestamp;
+  uint64_t total_lifetime;
+  uint32_t min_lifetime, max_lifetime;
+  uint32_t alloc_cpu_id, dealloc_cpu_id;
+  uint32_t num_migrated_cpu;
+
+  // Only compared to prior deallocated object currently.
+  uint32_t num_lifetime_overlaps;
+  uint32_t num_same_alloc_cpu;
+  uint32_t num_same_dealloc_cpu;
+
+  uint64_t data_type_id; // TODO: hash of type name
+
+  MemInfoBlock() : alloc_count(0) {}
+
+  MemInfoBlock(uint32_t size, uint64_t access_count, uint32_t alloc_timestamp,
+               uint32_t dealloc_timestamp, uint32_t alloc_cpu, uint32_t dealloc_cpu)
+      : alloc_count(1), total_access_count(access_count),
+        min_access_count(access_count), max_access_count(access_count),
+        total_size(size), min_size(size), max_size(size),
+        alloc_timestamp(alloc_timestamp), dealloc_timestamp(dealloc_timestamp),
+        total_lifetime(dealloc_timestamp - alloc_timestamp),
+        min_lifetime(total_lifetime), max_lifetime(total_lifetime),
+        alloc_cpu_id(alloc_cpu), dealloc_cpu_id(dealloc_cpu),
+        num_lifetime_overlaps(0), num_same_alloc_cpu(0),
+        num_same_dealloc_cpu(0) {
+    num_migrated_cpu = alloc_cpu_id != dealloc_cpu_id;
+  }
+
+  void Merge(const MemInfoBlock &newMIB) {
+    alloc_count += newMIB.alloc_count;
+
+    total_access_count += newMIB.total_access_count;
+    min_access_count = newMIB.min_access_count < min_access_count ? newMIB.min_access_count : min_access_count;
+    max_access_count = newMIB.max_access_count < max_access_count ? newMIB.max_access_count : max_access_count;
+
+    total_size += newMIB.total_size;
+    min_size = newMIB.min_size < min_size ? newMIB.min_size : min_size;
+    max_size = newMIB.max_size < max_size ? newMIB.max_size : max_size;
+
+    total_lifetime += newMIB.total_lifetime;
+    min_lifetime = newMIB.min_lifetime < min_lifetime ? newMIB.min_lifetime : min_lifetime;
+    max_lifetime = newMIB.max_lifetime > max_lifetime ? newMIB.max_lifetime : max_lifetime;
+
+    // We know newMIB was deallocated later, so just need to check if it was
+    // allocated before last one deallocated.
+    num_lifetime_overlaps += newMIB.alloc_timestamp < dealloc_timestamp;
+    alloc_timestamp = newMIB.alloc_timestamp;
+    dealloc_timestamp = newMIB.dealloc_timestamp;
+
+    num_same_alloc_cpu += alloc_cpu_id == newMIB.alloc_cpu_id;
+    num_same_dealloc_cpu += dealloc_cpu_id == newMIB.dealloc_cpu_id;
+    alloc_cpu_id = newMIB.alloc_cpu_id;
+    dealloc_cpu_id = newMIB.dealloc_cpu_id;
+  }
+});
+
 } // namespace memprof
 } // namespace llvm
 
 #endif

compiler-rt/lib/memprof/memprof_allocator.cpp

 //
 // Implementation of MemProf's memory allocator, which uses the allocator
 // from sanitizer_common.
 //
 //===----------------------------------------------------------------------===//
 
 #include "memprof_allocator.h"
 #include "memprof_mapping.h"
-#include "memprof_meminfoblock.h"
 #include "memprof_mibmap.h"
 #include "memprof_rawprofile.h"
 #include "memprof_stack.h"
 #include "memprof_thread.h"
+#include "profile/MemProfData.inc"
 #include "sanitizer_common/sanitizer_allocator_checks.h"
 #include "sanitizer_common/sanitizer_allocator_interface.h"
 #include "sanitizer_common/sanitizer_allocator_report.h"
 #include "sanitizer_common/sanitizer_errno.h"
 #include "sanitizer_common/sanitizer_file.h"
 #include "sanitizer_common/sanitizer_flags.h"
 #include "sanitizer_common/sanitizer_internal_defs.h"
 #include "sanitizer_common/sanitizer_list.h"
 #include "sanitizer_common/sanitizer_procmaps.h"
 #include "sanitizer_common/sanitizer_stackdepot.h"
 #include "sanitizer_common/sanitizer_vector.h"
 
 #include <sched.h>
 #include <time.h>
 
 namespace __memprof {
+namespace {
+using ::llvm::memprof::MemInfoBlock;
+
+void Print(const MemInfoBlock &M, const u64 id, bool print_terse) {
+  u64 p;
+
+  if (print_terse) {
+    p = M.total_size * 100 / M.alloc_count;
+    Printf("MIB:%llu/%u/%llu.%02llu/%u/%u/", id, M.alloc_count, p / 100,
+           p % 100, M.min_size, M.max_size);
+    p = M.total_access_count * 100 / M.alloc_count;
+    Printf("%llu.%02llu/%llu/%llu/", p / 100, p % 100, M.min_access_count,
+           M.max_access_count);
+    p = M.total_lifetime * 100 / M.alloc_count;
+    Printf("%llu.%02llu/%u/%u/", p / 100, p % 100, M.min_lifetime,
+           M.max_lifetime);
+    Printf("%u/%u/%u/%u\n", M.num_migrated_cpu, M.num_lifetime_overlaps,
+           M.num_same_alloc_cpu, M.num_same_dealloc_cpu);
+  } else {
+    p = M.total_size * 100 / M.alloc_count;
+    Printf("Memory allocation stack id = %llu\n", id);
+    Printf("\talloc_count %u, size (ave/min/max) %llu.%02llu / %u / %u\n",
+           M.alloc_count, p / 100, p % 100, M.min_size, M.max_size);
+    p = M.total_access_count * 100 / M.alloc_count;
+    Printf("\taccess_count (ave/min/max): %llu.%02llu / %llu / %llu\n", p / 100,
+           p % 100, M.min_access_count, M.max_access_count);
+    p = M.total_lifetime * 100 / M.alloc_count;
+    Printf("\tlifetime (ave/min/max): %llu.%02llu / %u / %u\n", p / 100,
+           p % 100, M.min_lifetime, M.max_lifetime);
+    Printf("\tnum migrated: %u, num lifetime overlaps: %u, num same alloc "
+           "cpu: %u, num same dealloc_cpu: %u\n",
+           M.num_migrated_cpu, M.num_lifetime_overlaps, M.num_same_alloc_cpu,
+           M.num_same_dealloc_cpu);
+  }
+}
+} // namespace
 
 static int GetCpuId(void) {
   // _memprof_preinit is called via the preinit_array, which subsequently calls
   // malloc. Since this is before _dl_init calls VDSO_SETUP, sched_getcpu
   // will seg fault as the address of __vdso_getcpu will be null.
   if (!memprof_init_done)
     return -1;
   return sched_getcpu();
struct Allocator {
   ~Allocator() {
     atomic_store_relaxed(&destructing, 1);
     FinishAndWrite();
   }
 
   static void PrintCallback(const uptr Key, LockedMemInfoBlock *const &Value,
                             void *Arg) {
     SpinMutexLock(&Value->mutex);
-    Value->mib.Print(Key, bool(Arg));
+    Print(Value->mib, Key, bool(Arg));
   }
 
   void FinishAndWrite() {
     if (print_text && common_flags()->print_module_map)
       DumpProcessMap();
 
     allocator.ForceLock();
 

compiler-rt/lib/memprof/memprof_meminfoblock.h

-#ifndef MEMPROF_MEMINFOBLOCK_H_
-#define MEMPROF_MEMINFOBLOCK_H_
-
-#include "memprof_interface_internal.h" // For u32, u64 TODO: Move these out of the internal header.
-#include "sanitizer_common/sanitizer_common.h"
-
-namespace __memprof {
-
-using __sanitizer::Printf;
-
-struct MemInfoBlock {
-  u32 alloc_count;
-  u64 total_access_count, min_access_count, max_access_count;
-  u64 total_size;
-  u32 min_size, max_size;
-  u32 alloc_timestamp, dealloc_timestamp;
-  u64 total_lifetime;
-  u32 min_lifetime, max_lifetime;
-  u32 alloc_cpu_id, dealloc_cpu_id;
-  u32 num_migrated_cpu;
-
-  // Only compared to prior deallocated object currently.
-  u32 num_lifetime_overlaps;
-  u32 num_same_alloc_cpu;
-  u32 num_same_dealloc_cpu;
-
-  u64 data_type_id; // TODO: hash of type name
-
-  MemInfoBlock() : alloc_count(0) {}
-
-  MemInfoBlock(u32 size, u64 access_count, u32 alloc_timestamp,
-               u32 dealloc_timestamp, u32 alloc_cpu, u32 dealloc_cpu)
-      : alloc_count(1), total_access_count(access_count),
-        min_access_count(access_count), max_access_count(access_count),
-        total_size(size), min_size(size), max_size(size),
-        alloc_timestamp(alloc_timestamp), dealloc_timestamp(dealloc_timestamp),
-        total_lifetime(dealloc_timestamp - alloc_timestamp),
-        min_lifetime(total_lifetime), max_lifetime(total_lifetime),
-        alloc_cpu_id(alloc_cpu), dealloc_cpu_id(dealloc_cpu),
-        num_lifetime_overlaps(0), num_same_alloc_cpu(0),
-        num_same_dealloc_cpu(0) {
-    num_migrated_cpu = alloc_cpu_id != dealloc_cpu_id;
-  }
-
-  void Print(u64 id, bool print_terse) const {
-    u64 p;
-
-    if (print_terse) {
-      p = total_size * 100 / alloc_count;
-      Printf("MIB:%llu/%u/%llu.%02llu/%u/%u/", id, alloc_count, p / 100,
-             p % 100, min_size, max_size);
-      p = total_access_count * 100 / alloc_count;
-      Printf("%llu.%02llu/%llu/%llu/", p / 100, p % 100, min_access_count,
-             max_access_count);
-      p = total_lifetime * 100 / alloc_count;
-      Printf("%llu.%02llu/%u/%u/", p / 100, p % 100, min_lifetime,
-             max_lifetime);
-      Printf("%u/%u/%u/%u\n", num_migrated_cpu, num_lifetime_overlaps,
-             num_same_alloc_cpu, num_same_dealloc_cpu);
-    } else {
-      p = total_size * 100 / alloc_count;
-      Printf("Memory allocation stack id = %llu\n", id);
-      Printf("\talloc_count %u, size (ave/min/max) %llu.%02llu / %u / %u\n",
-             alloc_count, p / 100, p % 100, min_size, max_size);
-      p = total_access_count * 100 / alloc_count;
-      Printf("\taccess_count (ave/min/max): %llu.%02llu / %llu / %llu\n",
-             p / 100, p % 100, min_access_count, max_access_count);
-      p = total_lifetime * 100 / alloc_count;
-      Printf("\tlifetime (ave/min/max): %llu.%02llu / %u / %u\n", p / 100,
-             p % 100, min_lifetime, max_lifetime);
-      Printf("\tnum migrated: %u, num lifetime overlaps: %u, num same alloc "
-             "cpu: %u, num same dealloc_cpu: %u\n",
-             num_migrated_cpu, num_lifetime_overlaps, num_same_alloc_cpu,
-             num_same_dealloc_cpu);
-    }
-  }
-
-  static void printHeader() {
-    Printf("MIB:StackID/AllocCount/AveSize/MinSize/MaxSize/AveAccessCount/"
-           "MinAccessCount/MaxAccessCount/AveLifetime/MinLifetime/MaxLifetime/"
-           "NumMigratedCpu/NumLifetimeOverlaps/NumSameAllocCpu/"
-           "NumSameDeallocCpu\n");
-  }
-
-  void Merge(const MemInfoBlock &newMIB) {
-    alloc_count += newMIB.alloc_count;
-
-    total_access_count += newMIB.total_access_count;
-    min_access_count = Min(min_access_count, newMIB.min_access_count);
-    max_access_count = Max(max_access_count, newMIB.max_access_count);
-
-    total_size += newMIB.total_size;
-    min_size = Min(min_size, newMIB.min_size);
-    max_size = Max(max_size, newMIB.max_size);
-
-    total_lifetime += newMIB.total_lifetime;
-    min_lifetime = Min(min_lifetime, newMIB.min_lifetime);
-    max_lifetime = Max(max_lifetime, newMIB.max_lifetime);
-
-    // We know newMIB was deallocated later, so just need to check if it was
-    // allocated before last one deallocated.
-    num_lifetime_overlaps += newMIB.alloc_timestamp < dealloc_timestamp;
-    alloc_timestamp = newMIB.alloc_timestamp;
-    dealloc_timestamp = newMIB.dealloc_timestamp;
-
-    num_same_alloc_cpu += alloc_cpu_id == newMIB.alloc_cpu_id;
-    num_same_dealloc_cpu += dealloc_cpu_id == newMIB.dealloc_cpu_id;
-    alloc_cpu_id = newMIB.alloc_cpu_id;
-    dealloc_cpu_id = newMIB.dealloc_cpu_id;
-  }
-
-} __attribute__((packed));
-
-} // namespace __memprof
-
-#endif // MEMPROF_MEMINFOBLOCK_H_

compiler-rt/lib/memprof/memprof_mibmap.h

 #ifndef MEMPROF_MIBMAP_H_
 #define MEMPROF_MIBMAP_H_
 
-#include "memprof_meminfoblock.h"
+#include <stdint.h>
+
+#include "profile/MemProfData.inc"
 #include "sanitizer_common/sanitizer_addrhashmap.h"
 #include "sanitizer_common/sanitizer_mutex.h"
 
 namespace __memprof {
 
 struct LockedMemInfoBlock {
   __sanitizer::StaticSpinMutex mutex;
-  MemInfoBlock mib;
+  ::llvm::memprof::MemInfoBlock mib;
 };
 
 // The MIB map stores a mapping from stack ids to MemInfoBlocks.
 typedef __sanitizer::AddrHashMap<LockedMemInfoBlock *, 200003> MIBMapTy;
 
 // Insert a new MemInfoBlock or merge with an existing block identified by the
 // stack id.
-void InsertOrMerge(const uptr Id, const MemInfoBlock &Block, MIBMapTy &Map);
+void InsertOrMerge(const uptr Id, const ::llvm::memprof::MemInfoBlock &Block,
+                   MIBMapTy &Map);
 
 } // namespace __memprof
 
 #endif // MEMPROF_MIBMAP_H_

compiler-rt/lib/memprof/memprof_mibmap.cpp

 //===-- memprof_mibmap.cpp -----------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file is a part of MemProfiler, a memory profiler.
 //
 //===----------------------------------------------------------------------===//
 
 #include "memprof_mibmap.h"
+#include "profile/MemProfData.inc"
 #include "sanitizer_common/sanitizer_allocator_internal.h"
 #include "sanitizer_common/sanitizer_mutex.h"
 
 namespace __memprof {
+using ::llvm::memprof::MemInfoBlock;
 
 void InsertOrMerge(const uptr Id, const MemInfoBlock &Block, MIBMapTy &Map) {
   MIBMapTy::Handle h(&Map, static_cast<uptr>(Id), /*remove=*/false,
                      /*create=*/true);
   if (h.created()) {
     LockedMemInfoBlock *lmib =
         (LockedMemInfoBlock *)InternalAlloc(sizeof(LockedMemInfoBlock));
     lmib->mutex.Init();

compiler-rt/lib/memprof/memprof_rawprofile.cpp

 #include <stdint.h>
 #include <stdlib.h>
 #include <string.h>
 
-#include "memprof_meminfoblock.h"
 #include "memprof_rawprofile.h"
 #include "profile/MemProfData.inc"
 #include "sanitizer_common/sanitizer_allocator_internal.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_linux.h"
 #include "sanitizer_common/sanitizer_procmaps.h"
 #include "sanitizer_common/sanitizer_stackdepot.h"
 #include "sanitizer_common/sanitizer_stackdepotbase.h"
 #include "sanitizer_common/sanitizer_stacktrace.h"
 #include "sanitizer_common/sanitizer_vector.h"
 
 namespace __memprof {
 using ::__sanitizer::Vector;
+using ::llvm::memprof::MemInfoBlock;
 using SegmentEntry = ::llvm::memprof::SegmentEntry;
 using Header = ::llvm::memprof::Header;
 
 namespace {
 template <class T> char *WriteBytes(T Pod, char *&Buffer) {
   *(T *)Buffer = Pod;
   return Buffer + sizeof(T);
 }
void SerializeSegmentsToBuffer(MemoryMappingLayoutBase &Layout,
   // Reserve space for the final count.
   Ptr += sizeof(u64);
 
   u64 NumSegmentsRecorded = 0;
   MemoryMappedSegment segment;
 
   for (Layout.Reset(); Layout.Next(&segment);) {
     if (segment.IsReadable() && segment.IsExecutable()) {
-      SegmentEntry Entry{};
-      Entry.Start = segment.start;
-      Entry.End = segment.end;
-      Entry.Offset = segment.offset;
-      memcpy(Entry.BuildId, segment.uuid, sizeof(segment.uuid));
+      // TODO: Record segment.uuid when it is implemented for Linux-Elf.
+      SegmentEntry Entry(segment.start, segment.end, segment.offset);
       memcpy(Ptr, &Entry, sizeof(SegmentEntry));
       Ptr += sizeof(SegmentEntry);
       NumSegmentsRecorded++;
     }
   }
 
   // Store the number of segments we recorded in the space we reserved.
   *((u64 *)Buffer) = NumSegmentsRecorded;

compiler-rt/lib/memprof/tests/rawprofile.cpp

 #include "memprof/memprof_rawprofile.h"
 
 #include <cstdint>
 #include <memory>
 
-#include "memprof/memprof_meminfoblock.h"
 #include "profile/MemProfData.inc"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_procmaps.h"
 #include "sanitizer_common/sanitizer_stackdepot.h"
 #include "sanitizer_common/sanitizer_stacktrace.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
 namespace {
 
-using ::__memprof::MemInfoBlock;
 using ::__memprof::MIBMapTy;
 using ::__memprof::SerializeToRawProfile;
 using ::__sanitizer::MemoryMappedSegment;
 using ::__sanitizer::MemoryMappingLayoutBase;
 using ::__sanitizer::StackDepotPut;
 using ::__sanitizer::StackTrace;
+using ::llvm::memprof::MemInfoBlock;
 using ::testing::_;
 using ::testing::Action;
 using ::testing::DoAll;
 using ::testing::Return;
 using ::testing::SetArgPointee;
 
 class MockMemoryMappingLayout final : public MemoryMappingLayoutBase {
 public:
   MOCK_METHOD(bool, Next, (MemoryMappedSegment *), (override));
   MOCK_METHOD(void, Reset, (), (override));
 };
 
-u64 PopulateFakeMap(const MemInfoBlock &FakeMIB, uptr StackPCBegin,
+uint64_t PopulateFakeMap(const MemInfoBlock &FakeMIB, uint64_t StackPCBegin,
                          MIBMapTy &FakeMap) {
   constexpr int kSize = 5;
-  uptr array[kSize];
+  uint64_t array[kSize];
   for (int i = 0; i < kSize; i++) {
     array[i] = StackPCBegin + i;
   }
   StackTrace St(array, kSize);
-  u32 Id = StackDepotPut(St);
+  uint32_t Id = StackDepotPut(St);
 
   InsertOrMerge(Id, FakeMIB, FakeMap);
   return Id;
 }
 
-template <class T = u64> T Read(char *&Buffer) {
+template <class T = uint64_t> T Read(char *&Buffer) {
   static_assert(std::is_pod<T>::value, "Must be a POD type.");
   assert(reinterpret_cast<size_t>(Buffer) % sizeof(T) == 0 &&
          "Unaligned read!");
   T t = *reinterpret_cast<T *>(Buffer);
   Buffer += sizeof(T);
   return t;
 }
 
TEST(MemProf, Basic) {
   MIBMapTy FakeMap;
   MemInfoBlock FakeMIB;
   // Since we want to override the constructor set vals to make it easier to
   // test.
   memset(&FakeMIB, 0, sizeof(MemInfoBlock));
   FakeMIB.alloc_count = 0x1;
   FakeMIB.total_access_count = 0x2;
 
-  u64 FakeIds[2];
+  uint64_t FakeIds[2];
   FakeIds[0] = PopulateFakeMap(FakeMIB, /*StackPCBegin=*/2, FakeMap);
   FakeIds[1] = PopulateFakeMap(FakeMIB, /*StackPCBegin=*/3, FakeMap);
 
   char *Ptr = nullptr;
-  u64 NumBytes = SerializeToRawProfile(FakeMap, Layout, Ptr);
+  uint64_t NumBytes = SerializeToRawProfile(FakeMap, Layout, Ptr);
   const char *Buffer = Ptr;
 
   ASSERT_GT(NumBytes, 0ULL);
   ASSERT_TRUE(Ptr);
 
   // Check the header.
   EXPECT_THAT(Read(Ptr), MEMPROF_RAW_MAGIC_64);
   EXPECT_THAT(Read(Ptr), MEMPROF_RAW_VERSION);
-  const u64 TotalSize = Read(Ptr);
-  const u64 SegmentOffset = Read(Ptr);
-  const u64 MIBOffset = Read(Ptr);
-  const u64 StackOffset = Read(Ptr);
+  const uint64_t TotalSize = Read(Ptr);
+  const uint64_t SegmentOffset = Read(Ptr);
+  const uint64_t MIBOffset = Read(Ptr);
+  const uint64_t StackOffset = Read(Ptr);
 
   // ============= Check sizes and padding.
   EXPECT_EQ(TotalSize, NumBytes);
   EXPECT_EQ(TotalSize % 8, 0ULL);
 
   // Should be equal to the size of the raw profile header.
   EXPECT_EQ(SegmentOffset, 48ULL);
 
   // We expect only 1 segment entry, 8b for the count and 56b for SegmentEntry
   // in memprof_rawprofile.cpp.
   EXPECT_EQ(MIBOffset - SegmentOffset, 64ULL);
 
   EXPECT_EQ(MIBOffset, 112ULL);
-  // We expect 2 mib entry, 8b for the count and sizeof(u64) +
+  // We expect 2 mib entry, 8b for the count and sizeof(uint64_t) +
   // sizeof(MemInfoBlock) contains stack id + MeminfoBlock.
   EXPECT_EQ(StackOffset - MIBOffset, 8 + 2 * (8 + sizeof(MemInfoBlock)));
 
   EXPECT_EQ(StackOffset, 336ULL);
   // We expect 2 stack entries, with 5 frames - 8b for total count,
   // 2 * (8b for id, 8b for frame count and 5*8b for fake frames).
   // Since this is the last section, there may be additional padding at the end
   // to make the total profile size 8b aligned.
   EXPECT_GE(TotalSize - StackOffset, 8ULL + 2 * (8 + 8 + 5 * 8));
 
   // ============= Check contents.
   unsigned char ExpectedSegmentBytes[64] = {
-      0x01, 0,   0,   0,   0,   0,   0, 0, // Number of entries
-      0x10, 0,   0,   0,   0,   0,   0, 0, // Start
-      0x20, 0,   0,   0,   0,   0,   0, 0, // End
-      0x10, 0,   0,   0,   0,   0,   0, 0, // Offset
-      0x0C, 0x0, 0xF, 0xF, 0xE, 0xE,       // Uuid
+      0x01, 0, 0, 0, 0, 0, 0, 0, // Number of entries
+      0x10, 0, 0, 0, 0, 0, 0, 0, // Start
+      0x20, 0, 0, 0, 0, 0, 0, 0, // End
+      0x10, 0, 0, 0, 0, 0, 0, 0, // Offset
+      0x0,                       // Uuid (not yet recorded).
   };
   EXPECT_EQ(memcmp(Buffer + SegmentOffset, ExpectedSegmentBytes, 64), 0);
 
   // Check that the number of entries is 2.
-  EXPECT_EQ(*reinterpret_cast<const u64 *>(Buffer + MIBOffset), 2ULL);
+  EXPECT_EQ(*reinterpret_cast<const uint64_t *>(Buffer + MIBOffset), 2ULL);
   // Check that stack id is set.
-  EXPECT_EQ(*reinterpret_cast<const u64 *>(Buffer + MIBOffset + 8), FakeIds[0]);
+  EXPECT_EQ(*reinterpret_cast<const uint64_t *>(Buffer + MIBOffset + 8),
+            FakeIds[0]);
 
   // Only check a few fields of the first MemInfoBlock.
   unsigned char ExpectedMIBBytes[sizeof(MemInfoBlock)] = {
       0x01, 0, 0, 0, // Alloc count
       0x02, 0, 0, 0, // Total access count
   };
   // Compare contents of 1st MIB after skipping count and stack id.
   EXPECT_EQ(
       memcmp(Buffer + MIBOffset + 16, ExpectedMIBBytes, sizeof(MemInfoBlock)),
       0);
   // Compare contents of 2nd MIB after skipping count and stack id for the first
   // and only the id for the second.
   EXPECT_EQ(memcmp(Buffer + MIBOffset + 16 + sizeof(MemInfoBlock) + 8,
                    ExpectedMIBBytes, sizeof(MemInfoBlock)),
             0);
 
   // Check that the number of entries is 2.
-  EXPECT_EQ(*reinterpret_cast<const u64 *>(Buffer + StackOffset), 2ULL);
+  EXPECT_EQ(*reinterpret_cast<const uint64_t *>(Buffer + StackOffset), 2ULL);
   // Check that the 1st stack id is set.
-  EXPECT_EQ(*reinterpret_cast<const u64 *>(Buffer + StackOffset + 8),
+  EXPECT_EQ(*reinterpret_cast<const uint64_t *>(Buffer + StackOffset + 8),
             FakeIds[0]);
   // Contents are num pcs, value of each pc - 1.
   unsigned char ExpectedStackBytes[2][6 * 8] = {
       {
           0x5, 0, 0, 0, 0, 0, 0, 0, // Number of PCs
           0x1, 0, 0, 0, 0, 0, 0, 0, // PC ...
           0x2, 0, 0, 0, 0, 0, 0, 0, 0x3, 0, 0, 0, 0, 0, 0, 0,
           0x4, 0, 0, 0, 0, 0, 0, 0, 0x5, 0, 0, 0, 0, 0, 0, 0,
       },
       {
           0x5, 0, 0, 0, 0, 0, 0, 0, // Number of PCs
           0x2, 0, 0, 0, 0, 0, 0, 0, // PC ...
           0x3, 0, 0, 0, 0, 0, 0, 0, 0x4, 0, 0, 0, 0, 0, 0, 0,
           0x5, 0, 0, 0, 0, 0, 0, 0, 0x6, 0, 0, 0, 0, 0, 0, 0,
       },
   };
   EXPECT_EQ(memcmp(Buffer + StackOffset + 16, ExpectedStackBytes[0],
                    sizeof(ExpectedStackBytes[0])),
             0);
 
   // Check that the 2nd stack id is set.
   EXPECT_EQ(
-      *reinterpret_cast<const u64 *>(Buffer + StackOffset + 8 + 6 * 8 + 8),
+      *reinterpret_cast<const uint64_t *>(Buffer + StackOffset + 8 + 6 * 8 + 8),
       FakeIds[1]);
 
   EXPECT_EQ(memcmp(Buffer + StackOffset + 16 + 6 * 8 + 8, ExpectedStackBytes[1],
                    sizeof(ExpectedStackBytes[1])),
             0);
 }
 
 } // namespace

llvm/include/llvm/ProfileData/MemProfData.inc

  * This file has two identical copies. The primary copy lives in LLVM and
  * the other one sits in compiler-rt/include/profile directory. To make changes
  * in this file, first modify the primary copy and copy it over to compiler-rt.
  * Testing of any change in this file can start only after the two copies are
  * synced up.
  *
 \*===----------------------------------------------------------------------===*/
 
-
 #ifdef _MSC_VER
-#define PACKED(__decl__) __pragma(pack(push,1)) __decl__ __pragma(pack(pop))
+#define PACKED(...) __pragma(pack(push,1)) __VA_ARGS__ __pragma(pack(pop))
 #else
-#define PACKED(__decl__) __decl__ __attribute__((__packed__))
+#define PACKED(...) __VA_ARGS__ __attribute__((__packed__))
 #endif
 
 // A 64-bit magic number to uniquely identify the raw binary memprof profile file.
 #define MEMPROF_RAW_MAGIC_64                                                                        \
   ((uint64_t)255 << 56 | (uint64_t)'m' << 48 | (uint64_t)'p' << 40 | (uint64_t)'r' << 32 |          \
    (uint64_t)'o' << 24 | (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129)
 
 // The version number of the raw binary format.
 #define MEMPROF_RAW_VERSION 1ULL
 
+#include <string.h>
+
 namespace llvm {
 namespace memprof {
 // A struct describing the header used for the raw binary memprof profile format.
 PACKED(struct Header {
   uint64_t Magic;
   uint64_t Version;
   uint64_t TotalSize;
   uint64_t SegmentOffset;
   uint64_t MIBOffset;
   uint64_t StackOffset;
 });
 
+
 // A struct describing the information necessary to describe a /proc/maps
 // segment entry for a particular binary/library identified by its build id.
 PACKED(struct SegmentEntry {
   uint64_t Start;
   uint64_t End;
   uint64_t Offset;
-  uint8_t BuildId[32];
+  // This field is unused until sanitizer procmaps support for build ids for
+  // Linux Elf is implemented.
+  uint8_t BuildId[32] = {0};
+
+  SegmentEntry(uint64_t S, uint64_t E, uint64_t O) :
+    Start(S), End(E), Offset(O) {}
+
+  SegmentEntry(const SegmentEntry& S) {
+    Start = S.Start;
+    End = S.End;
+    Offset = S.Offset;
+  }
+
+  SegmentEntry& operator=(const SegmentEntry& S) {
+    Start = S.Start;
+    End = S.End;
+    Offset = S.Offset;
+    return *this;
+  }
+
+  bool operator==(const SegmentEntry& S) const {
+    return Start == S.Start &&
+           End == S.End &&
+           Offset == S.Offset;
+  }
 });
+
+// A struct representing the heap allocation characteristics of a particular
+// runtime context. This struct is shared between the compiler-rt runtime and
+// the raw profile reader. The indexed format uses a separate, self-describing
+// backwards compatible format.
+PACKED(struct MemInfoBlock {
+  uint32_t alloc_count;
+  uint64_t total_access_count, min_access_count, max_access_count;
+  uint64_t total_size;
+  uint32_t min_size, max_size;
+  uint32_t alloc_timestamp, dealloc_timestamp;
+  uint64_t total_lifetime;
+  uint32_t min_lifetime, max_lifetime;
+  uint32_t alloc_cpu_id, dealloc_cpu_id;
+  uint32_t num_migrated_cpu;
+
+  // Only compared to prior deallocated object currently.
+  uint32_t num_lifetime_overlaps;
+  uint32_t num_same_alloc_cpu;
+  uint32_t num_same_dealloc_cpu;
+
+  uint64_t data_type_id; // TODO: hash of type name
+
+  MemInfoBlock() : alloc_count(0) {}
+
+  MemInfoBlock(uint32_t size, uint64_t access_count, uint32_t alloc_timestamp,
+               uint32_t dealloc_timestamp, uint32_t alloc_cpu, uint32_t dealloc_cpu)
+      : alloc_count(1), total_access_count(access_count),
+        min_access_count(access_count), max_access_count(access_count),
+        total_size(size), min_size(size), max_size(size),
+        alloc_timestamp(alloc_timestamp), dealloc_timestamp(dealloc_timestamp),
+        total_lifetime(dealloc_timestamp - alloc_timestamp),
+        min_lifetime(total_lifetime), max_lifetime(total_lifetime),
+        alloc_cpu_id(alloc_cpu), dealloc_cpu_id(dealloc_cpu),
+        num_lifetime_overlaps(0), num_same_alloc_cpu(0),
+        num_same_dealloc_cpu(0) {
+    num_migrated_cpu = alloc_cpu_id != dealloc_cpu_id;
+  }
+
+  void Merge(const MemInfoBlock &newMIB) {
+    alloc_count += newMIB.alloc_count;
+
+    total_access_count += newMIB.total_access_count;
+    min_access_count = newMIB.min_access_count < min_access_count ? newMIB.min_access_count : min_access_count;
+    max_access_count = newMIB.max_access_count < max_access_count ? newMIB.max_access_count : max_access_count;
+
+    total_size += newMIB.total_size;
+    min_size = newMIB.min_size < min_size ? newMIB.min_size : min_size;
+    max_size = newMIB.max_size < max_size ? newMIB.max_size : max_size;
+
+    total_lifetime += newMIB.total_lifetime;
+    min_lifetime = newMIB.min_lifetime < min_lifetime ? newMIB.min_lifetime : min_lifetime;
+    max_lifetime = newMIB.max_lifetime > max_lifetime ? newMIB.max_lifetime : max_lifetime;
+
+    // We know newMIB was deallocated later, so just need to check if it was
+    // allocated before last one deallocated.
+    num_lifetime_overlaps += newMIB.alloc_timestamp < dealloc_timestamp;
+    alloc_timestamp = newMIB.alloc_timestamp;
+    dealloc_timestamp = newMIB.dealloc_timestamp;
+
+    num_same_alloc_cpu += alloc_cpu_id == newMIB.alloc_cpu_id;
+    num_same_dealloc_cpu += dealloc_cpu_id == newMIB.dealloc_cpu_id;
+    alloc_cpu_id = newMIB.alloc_cpu_id;
+    dealloc_cpu_id = newMIB.dealloc_cpu_id;
+  }
+});
+
 } // namespace memprof
 } // namespace llvm
 
 #endif