[memprof] Update summary output.

[snehasish]

Summary

Update the YAML format print out of the profile to include a summary
instead of displaying the headers in the raw file buffer. This allows us
to release the raw buffer early saving memory.

[tejohnson]

I think the 2 changes are unrelated so maybe they should be committed separately? Question below about one piece of info that changed.

llvm/lib/ProfileData/RawMemProfReader.cpp

217

It looks like before we printed out the number of MIBs and now we print out the number of functions containing at least one alloc - is that intended? Should we print out both?

llvm/test/tools/llvm-profdata/memprof-multi.test

38

Comment seems stale unless the summary is changed to print number of MIBs again.

[snehasish]

Remove summary changes from this patch.
Add another field which tracks the number of mibs we have.

[snehasish]

PTAL, thanks!

llvm/lib/ProfileData/RawMemProfReader.cpp

217

Done, extended the logic to print out both.

llvm/test/tools/llvm-profdata/memprof-multi.test

38

Actually, in this case I added additional check lines below to ensure 2 MIBs are printed (symbolname, lineoffset and column). I extended the logic to also track the number of MIBs so that it's clearer.

[snehasish]

In D126834#3551932, @tejohnson wrote:

I think the 2 changes are unrelated so maybe they should be committed separately?

Split out the segment information print in D126840.

[tejohnson]

lgtm

llvm/include/llvm/ProfileData/RawMemProfReader.h

 namespace memprof {
 
 // Map from id (recorded from sanitizer stack depot) to virtual addresses for
 // each program counter address in the callstack.
 using CallStackMap = llvm::DenseMap<uint64_t, llvm::SmallVector<uint64_t>>;
 
 class RawMemProfReader {
 public:
-  RawMemProfReader(std::unique_ptr<MemoryBuffer> DataBuffer)
-      : DataBuffer(std::move(DataBuffer)) {}
   RawMemProfReader(const RawMemProfReader &) = delete;
   RawMemProfReader &operator=(const RawMemProfReader &) = delete;
 
   // Prints the contents of the profile in YAML format.
   void printYAML(raw_ostream &OS);
 
   // Return true if the \p DataBuffer starts with magic bytes indicating it is
   // a raw binary memprof profile.
public:
 
   // Return a const reference to the internal function profile data.
   const llvm::MapVector<GlobalValue::GUID, IndexedMemProfRecord> &
   getProfileData() const {
     return FunctionProfileData;
   }
 
 private:
-  RawMemProfReader(std::unique_ptr<MemoryBuffer> DataBuffer,
-                   object::OwningBinary<object::Binary> &&Bin, bool KeepName)
-      : DataBuffer(std::move(DataBuffer)), Binary(std::move(Bin)),
-        KeepSymbolName(KeepName) {}
-  Error initialize();
-  Error readRawProfile();
+  RawMemProfReader(object::OwningBinary<object::Binary> &&Bin, bool KeepName)
+      : Binary(std::move(Bin)), KeepSymbolName(KeepName) {}
+  // Initializes the RawMemProfReader with the contents in `DataBuffer`.
+  Error initialize(std::unique_ptr<MemoryBuffer> DataBuffer);
+  // Read and parse the contents of the `DataBuffer` as a binary format profile.
+  Error readRawProfile(std::unique_ptr<MemoryBuffer> DataBuffer);
   // Symbolize and cache all the virtual addresses we encounter in the
   // callstacks from the raw profile. Also prune callstack frames which we can't
   // symbolize or those that belong to the runtime. For profile entries where
   // the entire callstack is pruned, we drop the entry from the profile.
   Error symbolizeAndFilterStackFrames();
   // Construct memprof records for each function and store it in the
   // `FunctionProfileData` map. A function may have allocation profile data or
   // callsite data or both.
   Error mapRawProfileToRecords();
 
   // A helper method to extract the frame from the IdToFrame map.
   const Frame &idToFrame(const FrameId Id) const {
     auto It = IdToFrame.find(Id);
     assert(It != IdToFrame.end() && "Id not found in map.");
     return It->getSecond();
   }
 
   object::SectionedAddress getModuleOffset(uint64_t VirtualAddress);
-  // Prints aggregate counts for each raw profile parsed from the DataBuffer in
-  // YAML format.
-  void printSummaries(raw_ostream &OS) const;
 
-  std::unique_ptr<MemoryBuffer> DataBuffer;
   object::OwningBinary<object::Binary> Binary;
   std::unique_ptr<llvm::symbolize::SymbolizableModule> Symbolizer;
 
   // The contents of the raw profile.
   llvm::SmallVector<SegmentEntry, 16> SegmentInfo;
   // A map from callstack id (same as key in CallStackMap below) to the heap
   // information recorded for that allocation context.
   llvm::MapVector<uint64_t, MemInfoBlock> CallstackProfileData;

llvm/lib/ProfileData/RawMemProfReader.cpp

 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Path.h"
 
 #define DEBUG_TYPE "memprof"
 
 namespace llvm {
 namespace memprof {
 namespace {
-
-struct Summary {
-  uint64_t Version;
-  uint64_t TotalSizeBytes;
-  uint64_t NumSegments;
-  uint64_t NumMIBInfo;
-  uint64_t NumStackOffsets;
-};
-
 template <class T = uint64_t> inline T alignedRead(const char *Ptr) {
   static_assert(std::is_pod<T>::value, "Not a pod type.");
   assert(reinterpret_cast<size_t>(Ptr) % sizeof(T) == 0 && "Unaligned Read");
   return *reinterpret_cast<const T *>(Ptr);
 }
 
-Summary computeSummary(const char *Start) {
-  auto *H = reinterpret_cast<const Header *>(Start);
-
-  // Check alignment while reading the number of items in each section.
-  return Summary{
-      H->Version,
-      H->TotalSize,
-      alignedRead(Start + H->SegmentOffset),
-      alignedRead(Start + H->MIBOffset),
-      alignedRead(Start + H->StackOffset),
-  };
-}
-
 Error checkBuffer(const MemoryBuffer &Buffer) {
   if (!RawMemProfReader::hasFormat(Buffer))
     return make_error<InstrProfError>(instrprof_error::bad_magic);
 
   if (Buffer.getBufferSize() == 0)
     return make_error<InstrProfError>(instrprof_error::empty_raw_profile);
 
   if (Buffer.getBufferSize() < sizeof(Header)) {
return report(
         "Path to profiled binary is empty!");
 
   auto BinaryOr = llvm::object::createBinary(ProfiledBinary);
   if (!BinaryOr) {
     return report(BinaryOr.takeError(), ProfiledBinary);
   }
 
   // Use new here since constructor is private.
-  std::unique_ptr<RawMemProfReader> Reader(new RawMemProfReader(
-      std::move(Buffer), std::move(BinaryOr.get()), KeepName));
-  if (Error E = Reader->initialize()) {
+  std::unique_ptr<RawMemProfReader> Reader(
+      new RawMemProfReader(std::move(BinaryOr.get()), KeepName));
+  if (Error E = Reader->initialize(std::move(Buffer))) {
     return std::move(E);
   }
   return std::move(Reader);
 }
 
 bool RawMemProfReader::hasFormat(const StringRef Path) {
   auto BufferOr = MemoryBuffer::getFileOrSTDIN(Path);
   if (!BufferOr)
     return false;
 
   std::unique_ptr<MemoryBuffer> Buffer(BufferOr.get().release());
   return hasFormat(*Buffer);
 }
 
 bool RawMemProfReader::hasFormat(const MemoryBuffer &Buffer) {
   if (Buffer.getBufferSize() < sizeof(uint64_t))
     return false;
   // Aligned read to sanity check that the buffer was allocated with at least 8b
   // alignment.
   const uint64_t Magic = alignedRead(Buffer.getBufferStart());
   return Magic == MEMPROF_RAW_MAGIC_64;
 }
 
 void RawMemProfReader::printYAML(raw_ostream &OS) {
+  uint64_t NumAllocFunctions = 0, NumMibInfo = 0;
+  for (const auto &KV : FunctionProfileData) {
+    const size_t NumAllocSites = KV.second.AllocSites.size();
+    if (NumAllocSites > 0) {
+      NumAllocFunctions++;
+      NumMibInfo += NumAllocSites;
+    }
+  }
+
   OS << "MemprofProfile:\n";
-  // TODO: Update printSummaries to print out the data after the profile has
-  // been symbolized and pruned. We can parse some raw profile characteristics
-  // from the data buffer for additional information.
-  printSummaries(OS);
+  OS << "  Summary:\n";
+  OS << "    Version: " << MEMPROF_RAW_VERSION << "\n";
+  OS << "    NumSegments: " << SegmentInfo.size() << "\n";
+  OS << "    NumMibInfo: " << NumMibInfo << "\n";
+  OS << "    NumAllocFunctions: " << NumAllocFunctions << "\n";
+  OS << "    NumStackOffsets: " << StackMap.size() << "\n";
   // Print out the merged contents of the profiles.
   OS << "  Records:\n";
   for (const auto &Entry : *this) {
     OS << "  -\n";
     OS << "    FunctionGUID: " << Entry.first << "\n";
     Entry.second.print(OS);
   }
 }
 
-void RawMemProfReader::printSummaries(raw_ostream &OS) const {
-  const char *Next = DataBuffer->getBufferStart();
-  while (Next < DataBuffer->getBufferEnd()) {
-    auto Summary = computeSummary(Next);
-    OS << "  -\n";
-    OS << "  Header:\n";
-    OS << "    Version: " << Summary.Version << "\n";
-    OS << "    TotalSizeBytes: " << Summary.TotalSizeBytes << "\n";
-    OS << "    NumSegments: " << Summary.NumSegments << "\n";
-    OS << "    NumMibInfo: " << Summary.NumMIBInfo << "\n";
-    OS << "    NumStackOffsets: " << Summary.NumStackOffsets << "\n";
-    // TODO: Print the build ids once we can record them using the
-    // sanitizer_procmaps library for linux.
-
-    auto *H = reinterpret_cast<const Header *>(Next);
-    Next += H->TotalSize;
-  }
-}
-
-Error RawMemProfReader::initialize() {
+Error RawMemProfReader::initialize(std::unique_ptr<MemoryBuffer> DataBuffer) {
   const StringRef FileName = Binary.getBinary()->getFileName();
 
   auto *ElfObject = dyn_cast<object::ELFObjectFileBase>(Binary.getBinary());
   if (!ElfObject) {
     return report(make_error<StringError>(Twine("Not an ELF file: "),
                                           inconvertibleErrorCode()),
                   FileName);
   }
std::unique_ptr<DIContext> Context = DWARFContext::create(
       *Object, DWARFContext::ProcessDebugRelocations::Process);
 
   auto SOFOr = symbolize::SymbolizableObjectFile::create(
       Object, std::move(Context), /*UntagAddresses=*/false);
   if (!SOFOr)
     return report(SOFOr.takeError(), FileName);
   Symbolizer = std::move(SOFOr.get());
 
-  if (Error E = readRawProfile())
+  if (Error E = readRawProfile(std::move(DataBuffer)))
     return E;
 
   if (Error E = symbolizeAndFilterStackFrames())
     return E;
 
   return mapRawProfileToRecords();
 }
 
Error RawMemProfReader::symbolizeAndFilterStackFrames() {
   if (StackMap.empty())
     return make_error<InstrProfError>(
         instrprof_error::malformed,
         "no entries in callstack map after symbolization");
 
   return Error::success();
 }
 
-Error RawMemProfReader::readRawProfile() {
+Error RawMemProfReader::readRawProfile(
+    std::unique_ptr<MemoryBuffer> DataBuffer) {
   const char *Next = DataBuffer->getBufferStart();
 
   while (Next < DataBuffer->getBufferEnd()) {
     auto *Header = reinterpret_cast<const memprof::Header *>(Next);
 
     // Read in the segment information, check whether its the same across all
     // profiles in this binary file.
     const llvm::SmallVector<SegmentEntry> Entries =

llvm/test/tools/llvm-profdata/memprof-basic.test

 env MEMPROF_OPTIONS=log_path=stdout ./rawprofile.out > basic.memprofraw
 ```
 
 RUN: llvm-profdata show --memory %p/Inputs/basic.memprofraw --profiled-binary %p/Inputs/basic.memprofexe -o - | FileCheck %s
 
 We expect 2 MIB entries, 1 each for the malloc calls in the program. Any
 additional allocations which do not originate from the main binary are pruned.
 
-CHECK: MemprofProfile:
-CHECK-NEXT:   -
-CHECK-NEXT:   Header:
+CHECK:  MemprofProfile:
+CHECK-NEXT:   Summary:
 CHECK-NEXT:     Version: 1
-CHECK-NEXT:     TotalSizeBytes: 1016
 CHECK-NEXT:     NumSegments: 9
-CHECK-NEXT:     NumMibInfo: 3
-CHECK-NEXT:     NumStackOffsets: 3
+CHECK-NEXT:     NumMibInfo: 2
+CHECK-NEXT:     NumAllocFunctions: 1
+CHECK-NEXT:     NumStackOffsets: 2
 CHECK-NEXT:   Records:
 CHECK-NEXT:   -
 CHECK-NEXT:     FunctionGUID: {{[0-9]+}}
 CHECK-NEXT:     AllocSites:
 CHECK-NEXT:     -
 CHECK-NEXT:       Callstack:
 CHECK-NEXT:       -
 CHECK-NEXT:         Function: {{[0-9]+}}

llvm/test/tools/llvm-profdata/memprof-inline.test

bin/clang -fuse-ld=lld -Wl,--no-rosegment -gmlt -fdebug-info-for-profiling \
           -fno-optimize-sibling-calls -m64 -Wl,-build-id inline.c -o memprof-inline.exe
 
 env MEMPROF_OPTIONS=log_path=stdout ./memprof-inline.exe > inline.memprofraw
 ```
 
 RUN: llvm-profdata show --memory %p/Inputs/inline.memprofraw --profiled-binary %p/Inputs/inline.memprofexe | FileCheck %s
 
 CHECK:  MemprofProfile:
-CHECK-NEXT:  -
-CHECK-NEXT:  Header:
+CHECK-NEXT:  Summary:
 CHECK-NEXT:    Version: 1
-CHECK-NEXT:    TotalSizeBytes: 880
 CHECK-NEXT:    NumSegments: 9
 CHECK-NEXT:    NumMibInfo: 2
-CHECK-NEXT:    NumStackOffsets: 2
+CHECK-NEXT:    NumAllocFunctions: 2
+CHECK-NEXT:    NumStackOffsets: 1
 CHECK-NEXT:  Records:
 CHECK-NEXT:  -
 CHECK-NEXT:    FunctionGUID: 15505678318020221912
 CHECK-NEXT:    AllocSites:
 CHECK-NEXT:    -
 CHECK-NEXT:      Callstack:
 CHECK-NEXT:      -
 CHECK-NEXT:        Function: 15505678318020221912

llvm/test/tools/llvm-profdata/memprof-multi.test

clang -fuse-ld=lld -Wl,--no-rosegment -gmlt -fdebug-info-for-profiling \
       -fmemory-profile -mno-omit-leaf-frame-pointer -fno-omit-frame-pointer \
       -fno-optimize-sibling-calls -m64 -Wl,-build-id source.c -o multi.memprofexe
 
 env MEMPROF_OPTIONS=log_path=stdout ./rawprofile.out > multi.memprofraw
 ```
 
 RUN: llvm-profdata show --memory %p/Inputs/multi.memprofraw --profiled-binary %p/Inputs/multi.memprofexe -o - | FileCheck %s
 
 We expect 2 MIB entries, 1 each for the malloc calls in the program.
 
-CHECK: MemprofProfile:
-CHECK-NEXT:   -
-CHECK-NEXT:   Header:
-CHECK-NEXT:     Version: 1
-CHECK-NEXT:     TotalSizeBytes: 864
-CHECK-NEXT:     NumSegments: 9
-CHECK-NEXT:     NumMibInfo: 2
-CHECK-NEXT:     NumStackOffsets: 2
-CHECK-NEXT:   -
-CHECK-NEXT:   Header:
-CHECK-NEXT:     Version: 1
-CHECK-NEXT:     TotalSizeBytes: 864
-CHECK-NEXT:     NumSegments: 9
-CHECK-NEXT:     NumMibInfo: 2
-CHECK-NEXT:     NumStackOffsets: 2
+CHECK:  MemprofProfile:
+CHECK-NEXT:  Summary:
+CHECK-NEXT:    Version: 1
+CHECK-NEXT:    NumSegments: 9
+CHECK-NEXT:    NumMibInfo: 2
+CHECK-NEXT:    NumAllocFunctions: 1
+CHECK-NEXT:    NumStackOffsets: 2
+
+CHECK:        SymbolName: main
+CHECK-NEXT:     LineOffset: 1
+CHECK-NEXT:     Column: 21
+
+CHECK:        SymbolName: main
+CHECK-NEXT:     LineOffset: 5
+CHECK-NEXT:     Column: 15