[PGO] Add API to merge profile from buffer

[davidxl]

Summary

Notes: since VP is not supported with buffer API, the test case for VP merging is not added. The API to do profile/in-process counter matching check will be in a followup patch.

[vsk]

Lgtm with minor changes.

lib/profile/InstrProfiling.h

66

nit, from

lib/profile/InstrProfilingMerge.c

27

It's documented that the source/destination profile buffers should have the same layout, but we should return early if data_size(data_begin(), data_end()) != Header->DataSize. The same goes for CountersSize and NamesSize.

60

Please split this loop nest into a separate function.

make/platform/clang_linux.mk

83

nit, indentation

[silvas]

Hi David, this looks really nice. You were right that the profraw merging is quite simple.

Can you update instrprof-without-libc.c test to include __llvm_profile_merge_from_buffer?

Also, would it be reasonable to omit the value prof merging until we can test it? If not, that's fine, but it may simplify this patch.

test/profile/instrprof-merge.c

94

Nit: Note -> Not

[davidxl]

Addressed all review feedbacks from vsk and sean:

1. split the vp merger to avoid introduce libc dependency on buffer API client
2. added a test in instrprof-without-lib.c by calling buffer merge API
3. added a test case for VP data merging

lib/profile/CMakeLists.txt

 add_custom_target(profile)
 
 set(PROFILE_SOURCES
   GCDAProfiling.c
   InstrProfiling.c
   InstrProfilingValue.c
   InstrProfilingBuffer.c
   InstrProfilingFile.c
+  InstrProfilingMerge.c
   InstrProfilingWriter.c
   InstrProfilingPlatformDarwin.c
   InstrProfilingPlatformLinux.c
   InstrProfilingPlatformOther.c
   InstrProfilingRuntime.cc
   InstrProfilingUtil.c)
 
 if(WIN32)

lib/profile/InstrProfiling.h

 
 /*!
  * \brief Clear profile counters to zero.
  *
  */
 void __llvm_profile_reset_counters(void);
 
 /*!
+ * \brief Read profile data form buffer and merge with
+ * in-process profile counters. The client is expected to
+ * have checked or already knows the profile data in the
+ * buffer matches the in-process counter structure before
+ * calling it.
+ */
+void __llvm_profile_merge_from_buffer(const char *Profile, uint64_t Size);
+
+/*!
  * \brief Counts the number of times a target value is seen.
  *
  * Records the target value for the CounterIndex if not seen before. Otherwise,
  * increments the counter associated w/ the target value.
  * void __llvm_profile_instrument_target(uint64_t TargetValue, void *Data,
  *                                       uint32_t CounterIndex);
  */
 void INSTR_PROF_VALUE_PROF_FUNC(

lib/profile/InstrProfilingMerge.c

+/*===- InstrProfilingMerge.c - Profile in-process Merging  ---------------===*\
+|*
+|*                     The LLVM Compiler Infrastructure
+|*
+|* This file is distributed under the University of Illinois Open Source
+|* License. See LICENSE.TXT for details.
+|*
+\*===----------------------------------------------------------------------===*/
+
+#include "InstrProfiling.h"
+#include "InstrProfilingInternal.h"
+#include "InstrProfilingUtil.h"
+
+#define INSTR_PROF_VALUE_PROF_DATA
+#include "InstrProfData.inc"
+
+void __llvm_profile_merge_from_buffer(const char *ProfileData,
+                                      uint64_t ProfileSize) {
+  __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData;
+  __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData;
+  uint64_t *SrcCountersStart;
+  const char *SrcNameStart;
+  ValueProfData *SrcValueProfDataStart, *SrcValueProfData;
+
+  SrcDataStart =
+      (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header));
+  SrcDataEnd = SrcDataStart + Header->DataSize;
+  SrcCountersStart = (uint64_t *)SrcDataEnd;
+  SrcNameStart = (const char *)(SrcCountersStart + Header->CountersSize);
+  SrcValueProfDataStart =
+      (ValueProfData *)(SrcNameStart + Header->NamesSize +
+                        __llvm_profile_get_num_padding_bytes(
+                            Header->NamesSize));
+
+  for (SrcData = SrcDataStart,
+      DstData = (__llvm_profile_data *)__llvm_profile_begin_data(),
+      SrcValueProfData = SrcValueProfDataStart;
+       SrcData < SrcDataEnd; ++SrcData, ++DstData) {
+    uint64_t *SrcCounters;
+    uint64_t *DstCounters = (uint64_t *)DstData->CounterPtr;
+    unsigned I, S, V, C, NC, NVK = 0;
+    ValueProfRecord *VR;
+    InstrProfValueData *VData;
+
+    NC = SrcData->NumCounters;
+    SrcCounters = SrcCountersStart +
+                  ((size_t)SrcData->CounterPtr - Header->CountersDelta) /
+                      sizeof(uint64_t);
+    for (I = 0; I < NC; I++)
+      DstCounters[I] += SrcCounters[I];
+
+    /* Now merge value profile data.  */
+    for (I = 0; I <= IPVK_Last; I++)
+      NVK += (SrcData->NumValueSites[I] != 0);
+
+    if (!NVK)
+      continue;
+
+    VR = getFirstValueProfRecord(SrcValueProfData);
+    for (I = 0; I < SrcValueProfData->NumValueKinds; I++) {
+      VData = getValueProfRecordValueData(VR);
+      for (S = 0; S < VR->NumValueSites; S++) {
+        uint8_t NV = VR->SiteCountArray[S];
+        for (V = 0; V < NV; V++) {
+          for (C = 0; C < VData[V].Count; C++)
+            __llvm_profile_instrument_target(VData[V].Value, DstData, S);
+        }
+      }
+      VR = getValueProfRecordNext(VR);
+    }
+    SrcValueProfData = (ValueProfData *)((char *)SrcValueProfData +
+                                         SrcValueProfData->TotalSize);
+  }
+}

make/platform/clang_linux.mk

 CFLAGS.profile-i386 := $(CFLAGS) -m32
 CFLAGS.profile-x86_64 := $(CFLAGS) -m64
 
 FUNCTIONS.builtins-i386 := $(CommonFunctions) $(ArchFunctions.i386)
 FUNCTIONS.builtins-x86_64 := $(CommonFunctions) $(ArchFunctions.x86_64)
 FUNCTIONS.profile-i386 := GCDAProfiling InstrProfiling InstrProfilingBuffer \
                           InstrProfilingFile InstrProfilingPlatformOther \
                           InstrProfilingRuntime InstrProfilingUtil \
-                          InstrProfilingWriter InstrProfilingValue
+                          InstrProfilingWriter InstrProfilingValue \
+        InstrProfilingMerge
 FUNCTIONS.profile-x86_64 := $(FUNCTIONS.profile-i386)
 
 # Always use optimized variants.
 OPTIMIZED := 1
 
 # We don't need to use visibility hidden on Linux.
 VISIBILITY_HIDDEN := 0
 
 SHARED_LIBRARY_SUFFIX := so

test/profile/instrprof-merge.c

+// RUN: %clang_profgen -O2 -o %t %s
+// RUN: %run %t %t.profraw 1 1
+// RUN: llvm-profdata show --all-functions --counts %t.profraw  | FileCheck %s
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+int __llvm_profile_runtime = 0;
+uint64_t __llvm_profile_get_size_for_buffer(void);
+int __llvm_profile_write_buffer(char *);
+void __llvm_profile_reset_counters(void);
+void __llvm_profile_merge_from_buffer(const char*, uint64_t);
+
+int dumpBuffer(const char *FileN, const char *Buffer, uint64_t Size)  {
+  FILE *File = fopen(FileN, "w");
+  if (!File)
+    return 1;
+  if (fwrite(Buffer, 1, Size, File) != Size)
+    return 1;
+  return fclose(File);
+}
+
+int g = 0;
+void foo(char c) {
+  if (c == '1')
+    g++;
+  else 
+    g--;
+}
+
+/* This function is not profiled */
+void bar(int M) {
+    g += M;
+}
+
+int main(int argc, const char *argv[]) {
+  int i;
+  if (argc < 4)
+    return 1;
+
+  const uint64_t MaxSize = 10000;
+  static char Buffer[MaxSize];
+
+  uint64_t Size = __llvm_profile_get_size_for_buffer();
+  if (Size > MaxSize)
+    return 1;
+
+  /* Start profiling. */
+  __llvm_profile_reset_counters();
+  foo(argv[2][0]);
+  /* End profiling by freezing counters*/
+  if (__llvm_profile_write_buffer(Buffer))
+    return 1;
+
+  /* Its profile will be discarded. */
+  for (i = 0; i < 10; i++)
+    bar(1);
+
+  /* Start profiling again and merge in previously
+   * saved counters in buffer. */
+  __llvm_profile_reset_counters();
+  __llvm_profile_merge_from_buffer(Buffer, Size);
+  foo(argv[3][0]);
+  /* End profiling */
+  if (__llvm_profile_write_buffer(Buffer))
+    return 1;
+
+  /* Its profile will be discarded. */
+  bar(2);
+
+  /* Now it is time to dump the profile to file.  */
+  return dumpBuffer(argv[0], Buffer, Size);
+}
+
+// Not profiled
+// CHECK-LABEL: dumpBuffer:
+// CHECK:        Counters: 3
+// CHECK-NEXT:   Function count: 0
+// CHECK-NEXT:   Block counts: [0, 0]
+
+// Profiled with entry count == 2
+// CHECK-LABEL:  foo:
+// CHECK:         Counters: 2
+// CHECK-NEXT:    Function count: 2
+// CHECK-NEXT:    Block counts: [2]
+
+// Not profiled
+// CHECK-LABEL:  bar:
+// CHECK:         Counters: 1
+// CHECK-NEXT     Function count: 0
+// CHECK-NEXT     Block counts: []
+
+// Note profiled
+// CHECK-LABEL:  main:
+// CHECK:         Counters: 6
+// CHECK-NEXT:    Function count: 0
+// CHECK-NEXT:    Block counts: [0, 0, 0, 0, 0]
+