[PGO] Add API to merge profile from buffer

[davidxl]

Summary

Notes: since VP is not supported with buffer API, the test case for VP merging is not added. The API to do profile/in-process counter matching check will be in a followup patch.

[vsk]

Lgtm with minor changes.

lib/profile/InstrProfiling.h

66

nit, from

lib/profile/InstrProfilingMerge.c

28

It's documented that the source/destination profile buffers should have the same layout, but we should return early if data_size(data_begin(), data_end()) != Header->DataSize. The same goes for CountersSize and NamesSize.

61

Please split this loop nest into a separate function.

make/platform/clang_linux.mk

83

nit, indentation

[silvas]

Hi David, this looks really nice. You were right that the profraw merging is quite simple.

Can you update instrprof-without-libc.c test to include __llvm_profile_merge_from_buffer?

Also, would it be reasonable to omit the value prof merging until we can test it? If not, that's fine, but it may simplify this patch.

test/profile/instrprof-merge.c

95

Nit: Note -> Not

[davidxl]

Addressed all review feedbacks from vsk and sean:

1. split the vp merger to avoid introduce libc dependency on buffer API client
2. added a test in instrprof-without-lib.c by calling buffer merge API
3. added a test case for VP data merging

lib/profile/CMakeLists.txt

   InstrProfilingValue.c
   InstrProfilingBuffer.c
   InstrProfilingFile.c
+  InstrProfilingMerge.c
+  InstrProfilingMergeFile.c
   InstrProfilingWriter.c
   InstrProfilingPlatformDarwin.c
   InstrProfilingPlatformLinux.c

lib/profile/InstrProfiling.h

 void __llvm_profile_reset_counters(void);
 
 /*!
+ * \brief Read profile data form buffer and merge with
+ * in-process profile counters. The client is expected to
+ * have checked or already knows the profile data in the
+ * buffer matches the in-process counter structure before
+ * calling it.
+ */
+void __llvm_profile_merge_from_buffer(const char *Profile, uint64_t Size);
+
+/*!
  * \brief Counts the number of times a target value is seen.
  *
  * Records the target value for the CounterIndex if not seen before. Otherwise,

lib/profile/InstrProfilingInternal.h

                           struct ValueProfData **ValueDataBeginArray,
                           const uint64_t ValueDataSize, const char *NamesBegin,
                           const char *NamesEnd);
+/* Merge value profile data pointed to by SrcValueProfData into
+ * in-memory profile counters pointed by to DstData.  */
+void mergeValueProfData(struct ValueProfData *SrcValueProfData,
+                        __llvm_profile_data *DstData);
 
 extern char *(*GetEnvHook)(const char *);
 extern void (*FreeHook)(void *);
-extern void* (*CallocHook)(size_t, size_t);
+extern void *(*CallocHook)(size_t, size_t);
+extern void (*VPMergeHook)(struct ValueProfData *, __llvm_profile_data *);
 extern uint32_t VPBufferSize;
 
 #endif

lib/profile/InstrProfilingMerge.c

+/*===- InstrProfilingMerge.c - Profile in-process Merging  ---------------===*\
+|*
+|*                     The LLVM Compiler Infrastructure
+|*
+|* This file is distributed under the University of Illinois Open Source
+|* License. See LICENSE.TXT for details.
+|*
+|*===----------------------------------------------------------------------===*
+|* This file defines the API needed for in-process merging of profile data
+|* stored in memory buffer.
+\*===---------------------------------------------------------------------===*/
+
+#include "InstrProfiling.h"
+#include "InstrProfilingInternal.h"
+#include "InstrProfilingUtil.h"
+
+#define INSTR_PROF_VALUE_PROF_DATA
+#include "InstrProfData.inc"
+
+COMPILER_RT_WEAK void (*VPMergeHook)(ValueProfData *,
+                                     __llvm_profile_data *) = NULL;
+
+void __llvm_profile_merge_from_buffer(const char *ProfileData,
+                                      uint64_t ProfileSize) {
+  __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData;
+  __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData;
+  uint64_t *SrcCountersStart;
+  const char *SrcNameStart;
+  ValueProfData *SrcValueProfDataStart, *SrcValueProfData;
+
+  SrcDataStart =
+      (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header));
+  SrcDataEnd = SrcDataStart + Header->DataSize;
+  SrcCountersStart = (uint64_t *)SrcDataEnd;
+  SrcNameStart = (const char *)(SrcCountersStart + Header->CountersSize);
+  SrcValueProfDataStart =
+      (ValueProfData *)(SrcNameStart + Header->NamesSize +
+                        __llvm_profile_get_num_padding_bytes(
+                            Header->NamesSize));
+
+  for (SrcData = SrcDataStart,
+      DstData = (__llvm_profile_data *)__llvm_profile_begin_data(),
+      SrcValueProfData = SrcValueProfDataStart;
+       SrcData < SrcDataEnd; ++SrcData, ++DstData) {
+    uint64_t *SrcCounters;
+    uint64_t *DstCounters = (uint64_t *)DstData->CounterPtr;
+    unsigned I, NC, NVK = 0;
+
+    NC = SrcData->NumCounters;
+    SrcCounters = SrcCountersStart +
+                  ((size_t)SrcData->CounterPtr - Header->CountersDelta) /
+                      sizeof(uint64_t);
+    for (I = 0; I < NC; I++)
+      DstCounters[I] += SrcCounters[I];
+
+    /* Now merge value profile data.  */
+    if (!VPMergeHook)
+      continue;
+
+    for (I = 0; I <= IPVK_Last; I++)
+      NVK += (SrcData->NumValueSites[I] != 0);
+
+    if (!NVK)
+      continue;
+
+    VPMergeHook(SrcValueProfData, DstData);
+    SrcValueProfData = (ValueProfData *)((char *)SrcValueProfData +
+                                         SrcValueProfData->TotalSize);
+  }
+}

lib/profile/InstrProfilingMergeFile.c

+/*===- InstrProfilingMergeFile.c - Profile in-process Merging  ------------===*\
+|*
+|*                     The LLVM Compiler Infrastructure
+|*
+|* This file is distributed under the University of Illinois Open Source
+|* License. See LICENSE.TXT for details.
+|*
+|*===----------------------------------------------------------------------===
+|* This file defines APIs needed to support in-process merging for profile data
+|* stored in files.
+\*===----------------------------------------------------------------------===*/
+
+#include "InstrProfiling.h"
+#include "InstrProfilingInternal.h"
+#include "InstrProfilingUtil.h"
+
+#define INSTR_PROF_VALUE_PROF_DATA
+#include "InstrProfData.inc"
+
+void (*VPMergeHook)(ValueProfData *,
+                    __llvm_profile_data *) = &mergeValueProfData;
+
+/* Merge value profile data pointed to by SrcValueProfData into
+ * in-memory profile counters pointed by to DstData.  */
+void mergeValueProfData(ValueProfData *SrcValueProfData,
+                        __llvm_profile_data *DstData) {
+  unsigned I, S, V, C;
+  InstrProfValueData *VData;
+  ValueProfRecord *VR = getFirstValueProfRecord(SrcValueProfData);
+  for (I = 0; I < SrcValueProfData->NumValueKinds; I++) {
+    VData = getValueProfRecordValueData(VR);
+    for (S = 0; S < VR->NumValueSites; S++) {
+      uint8_t NV = VR->SiteCountArray[S];
+      for (V = 0; V < NV; V++) {
+        for (C = 0; C < VData[V].Count; C++)
+          __llvm_profile_instrument_target(VData[V].Value, DstData, S);
+      }
+    }
+    VR = getValueProfRecordNext(VR);
+  }
+}

make/platform/clang_linux.mk

 FUNCTIONS.profile-i386 := GCDAProfiling InstrProfiling InstrProfilingBuffer \
                           InstrProfilingFile InstrProfilingPlatformOther \
                           InstrProfilingRuntime InstrProfilingUtil \
-                          InstrProfilingWriter InstrProfilingValue
+                          InstrProfilingWriter InstrProfilingValue \
+                          InstrProfilingMerge InstrProfilingMergeFile
 FUNCTIONS.profile-x86_64 := $(FUNCTIONS.profile-i386)
 
 # Always use optimized variants.

test/profile/Linux/instrprof-merge-vp.c

+// RUN: %clang_profgen -mllvm --enable-value-profiling=true -O2 -o %t %s
+// RUN: %run %t %t.profraw
+// RUN: llvm-profdata merge -o %t.profdata %t.profraw
+// RUN: llvm-profdata show --all-functions --counts --ic-targets %t.profdata | FileCheck %s
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+int __llvm_profile_runtime = 0;
+int __llvm_profile_write_file();
+void __llvm_profile_reset_counters(void);
+void __llvm_profile_merge_from_buffer(const char *, uint64_t);
+void __llvm_profile_set_filename(const char *);
+struct __llvm_profile_data;
+struct ValueProfData;
+void mergeValueProfData(struct ValueProfData *, struct __llvm_profile_data *);
+/* Force the vp merger module to be linked in.  */
+void *Dummy = &mergeValueProfData;
+
+void callee1() {}
+void callee2() {}
+void callee3() {}
+
+typedef void (*FP)(void);
+FP Fps[3] = {callee1, callee2, callee3};
+
+void foo(int N) {
+  int I, J;
+  for (I = 0; I < 3; I++)
+    for (J = 0; J < I * 2 + 1; J++)
+      Fps[I]();
+
+  if (N < 2)
+    return;
+
+  for (I = 0; I < 3; I++)
+    for (J = 0; J < I * 2 + 1; J++)
+      Fps[2 - I]();
+}
+
+/* This function is not profiled */
+void bar(void) {
+  int I;
+  for (I = 0; I < 20; I++)
+    Fps[I % 3]();
+}
+
+int main(int argc, const char *argv[]) {
+  int i;
+  if (argc < 2)
+    return 1;
+
+  const char *FileN = argv[1];
+  __llvm_profile_set_filename(FileN);
+  /* Start profiling. */
+  __llvm_profile_reset_counters();
+  foo(1);
+  /* End profiling by freezing counters and
+   * dump them to the file. */
+  if (__llvm_profile_write_file())
+    return 1;
+
+  /* Read profile data into buffer. */
+  FILE *File = fopen(FileN, "r");
+  if (!File)
+    return 1;
+  fseek(File, 0, SEEK_END);
+  uint64_t Size = ftell(File);
+  fseek(File, 0, SEEK_SET);
+  char *Buffer = (char *)malloc(Size);
+  if (Size != fread(Buffer, 1, Size, File))
+    return 1;
+  fclose(File);
+
+  /* Its profile will be discarded. */
+  for (i = 0; i < 10; i++)
+    bar();
+
+  /* Start profiling again and merge in previously
+     saved counters in buffer. */
+  __llvm_profile_reset_counters();
+  __llvm_profile_merge_from_buffer(Buffer, Size);
+  foo(2);
+  /* End profiling. */
+  truncate(FileN, 0);
+  if (__llvm_profile_write_file())
+    return 1;
+
+  /* Its profile will be discarded. */
+  bar();
+
+  return 0;
+}
+
+// CHECK-LABEL:  foo:
+// CHECK:    Indirect Target Results:
+// CHECK-NEXT:  [ 0, callee3, 10 ]
+// CHECK-NEXT:  [ 0, callee2, 6 ]
+// CHECK-NEXT:  [ 0, callee1, 2 ]
+// CHECK-NEXT:  [ 1, callee1, 5 ]
+// CHECK-NEXT:  [ 1, callee2, 3 ]
+// CHECK-NEXT:  [ 1, callee3, 1 ]
+
+// CHECK-LABEL: bar:
+// CHECK:         [ 0, callee1, 0 ]
+// CHECK-NEXT:    [ 0, callee2, 0 ]
+// CHECK-NEXT:    [ 0, callee3, 0 ]

test/profile/instrprof-merge.c

+// RUN: %clang_profgen -O2 -o %t %s
+// RUN: %run %t %t.profraw 1 1
+// RUN: llvm-profdata show --all-functions --counts %t.profraw  | FileCheck %s
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+int __llvm_profile_runtime = 0;
+uint64_t __llvm_profile_get_size_for_buffer(void);
+int __llvm_profile_write_buffer(char *);
+void __llvm_profile_reset_counters(void);
+void __llvm_profile_merge_from_buffer(const char *, uint64_t);
+
+int dumpBuffer(const char *FileN, const char *Buffer, uint64_t Size) {
+  FILE *File = fopen(FileN, "w");
+  if (!File)
+    return 1;
+  if (fwrite(Buffer, 1, Size, File) != Size)
+    return 1;
+  return fclose(File);
+}
+
+int g = 0;
+void foo(char c) {
+  if (c == '1')
+    g++;
+  else
+    g--;
+}
+
+/* This function is not profiled */
+void bar(int M) { g += M; }
+
+int main(int argc, const char *argv[]) {
+  int i;
+  if (argc < 4)
+    return 1;
+
+  const uint64_t MaxSize = 10000;
+  static char Buffer[MaxSize];
+
+  uint64_t Size = __llvm_profile_get_size_for_buffer();
+  if (Size > MaxSize)
+    return 1;
+
+  /* Start profiling. */
+  __llvm_profile_reset_counters();
+  foo(argv[2][0]);
+  /* End profiling by freezing counters. */
+  if (__llvm_profile_write_buffer(Buffer))
+    return 1;
+
+  /* Its profile will be discarded. */
+  for (i = 0; i < 10; i++)
+    bar(1);
+
+  /* Start profiling again and merge in previously
+     saved counters in buffer. */
+  __llvm_profile_reset_counters();
+  __llvm_profile_merge_from_buffer(Buffer, Size);
+  foo(argv[3][0]);
+  /* End profiling */
+  if (__llvm_profile_write_buffer(Buffer))
+    return 1;
+
+  /* Its profile will be discarded. */
+  bar(2);
+
+  /* Now it is time to dump the profile to file.  */
+  return dumpBuffer(argv[1], Buffer, Size);
+}
+
+// Not profiled
+// CHECK-LABEL: dumpBuffer:
+// CHECK:        Counters: 3
+// CHECK-NEXT:   Function count: 0
+// CHECK-NEXT:   Block counts: [0, 0]
+
+// Profiled with entry count == 2
+// CHECK-LABEL:  foo:
+// CHECK:         Counters: 2
+// CHECK-NEXT:    Function count: 2
+// CHECK-NEXT:    Block counts: [2]
+
+// Not profiled
+// CHECK-LABEL:  bar:
+// CHECK:         Counters: 1
+// CHECK-NEXT     Function count: 0
+// CHECK-NEXT     Block counts: []
+
+// Not profiled
+// CHECK-LABEL:  main:
+// CHECK:         Counters: 6
+// CHECK-NEXT:    Function count: 0
+// CHECK-NEXT:    Block counts: [0, 0, 0, 0, 0]

test/profile/instrprof-without-libc.c

 int __llvm_profile_runtime = 0;
 uint64_t __llvm_profile_get_size_for_buffer(void);
 int __llvm_profile_write_buffer(char *);
+void __llvm_profile_merge_from_buffer(const char *, uint64_t Size);
+
 int write_buffer(uint64_t, const char *);
 int main(int argc, const char *argv[]) {
   // CHECK-LABEL: define {{.*}} @main(

 #ifdef CHECK_SYMBOLS
   // Don't write it out.  Since we're checking the symbols, we don't have libc
   // available.
+  // Call merge function to make sure it does not bring in libc deps:
+  __llvm_profile_merge_from_buffer(Buffer, Size);
   return 0;
 #else
   // Actually write it out so we can FileCheck the output.