Order File Instrumentation: dump the data in compiler-rt

[manmanren]

Small Nits

[davidxl]

lib/profile/InstrProfilingFile.c

690

I suggest not using the main file for order dumping for a couple of reasons:

1. To append it into the main file, you will need to update the raw header to set order file size
2. you will need to add profile merging support for ordering file. It can be tricky as the value profile data length is variable from run to run

To dump into a separate file, you can use the main filename as the base with a differnent suffix:

main file: default_xxx.profraw
order file: default_xxx.order, or simply default_xxx.profraw.order

[manmanren]

Hi David,

Thanks for reviewing the patch! Sorry for the delay! Just came back from a vacation.

The updated diff dumps order file profile data in xxx.order.

Manman

[davidxl]

thanks. Can you add a few test cases in the patch?

[manmanren]

Add a testing case to make sure the instrumentation works correctly!

[davidxl]

lib/profile/InstrProfilingFile.c

115

As follow up, I think it is better to make buffer size controllable by an internal option, and compiler can pass the size to runtime via a variable. Not needed for this patch.

116

The #if 0 needs to be cleaned up.

test/profile/Inputs/instrprof-order-file.c

11

Perfhaps call 'g' in a loop to show only one entry is recorded?

12

add another call to 'f' after call to 'g' to test it is not double recorded.

[manmanren]

Address review comments!

[davidxl]

test/profile/instrprof-order-file.test

5

is this option -forder-file-instrumentation already available in clang?

7

is 'od' portable? maybe exclude it for windows?

[manmanren]

Thanks!

Manman

lib/profile/InstrProfilingFile.c

116

Oh that was when I was debugging! Sorry about that.

test/profile/instrprof-order-file.test

5

This is in a separate diff (D58751) :]

7

Good point!

[manmanren]

Mark windows as unsupported in the testing case.

[davidxl]

lgtm

lib/profile/InstrProfData.inc

INSTR_PROF_SECT_ENTRY(IPSK_vals, \
                       INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \
                       INSTR_PROF_VALS_COFF, "__DATA,")
 INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \
                       INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \
                       INSTR_PROF_VNODES_COFF, "__DATA,")
 INSTR_PROF_SECT_ENTRY(IPSK_covmap, \
                       INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \
                       INSTR_PROF_COVMAP_COFF, "__LLVM_COV,")
+INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \
+                      INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \
+                      INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,")
 
 #undef INSTR_PROF_SECT_ENTRY
 #endif
 
 
 #ifdef INSTR_PROF_VALUE_PROF_DATA
 #define INSTR_PROF_DATA_DEFINED
 

 /* section name strings common to all targets other
    than WIN32 */
 #define INSTR_PROF_DATA_COMMON __llvm_prf_data
 #define INSTR_PROF_NAME_COMMON __llvm_prf_names
 #define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts
 #define INSTR_PROF_VALS_COMMON __llvm_prf_vals
 #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
 #define INSTR_PROF_COVMAP_COMMON __llvm_covmap
+#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile
 /* Windows section names. Because these section names contain dollar characters,
  * they must be quoted.
  */
 #define INSTR_PROF_DATA_COFF ".lprfd$M"
 #define INSTR_PROF_NAME_COFF ".lprfn$M"
 #define INSTR_PROF_CNTS_COFF ".lprfc$M"
 #define INSTR_PROF_VALS_COFF ".lprfv$M"
 #define INSTR_PROF_VNODES_COFF ".lprfnd$M"
 #define INSTR_PROF_COVMAP_COFF ".lcovmap$M"
+#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M"
 
 #ifdef _WIN32
 /* Runtime section names and name strings.  */
 #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF
 #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF
 #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF
 /* Array of pointers. Each pointer points to a list
  * of value nodes associated with one value site.
  */
 #define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF
 /* Value profile nodes section. */
 #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF
 #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF
+#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF
 #else
 /* Runtime section names and name strings.  */
 #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON)
 #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON)
 #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON)
 /* Array of pointers. Each pointer points to a list
  * of value nodes associated with one value site.
  */
 #define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON)
 /* Value profile nodes section. */
 #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON)
 #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON)
+/* Order file instrumentation. */
+#define INSTR_PROF_ORDERFILE_SECT_NAME                                         \
+  INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON)
 #endif
 
+#define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer
+#define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR                                   \
+  INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME)
+#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx
+#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR                               \
+  INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME)
+
 /* Macros to define start/stop section symbol for a given
  * section on Linux. For instance
  * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will
  * expand to __start___llvm_prof_data
  */
 #define INSTR_PROF_SECT_START(Sect) \
         INSTR_PROF_CONCAT(__start_,Sect)
 #define INSTR_PROF_SECT_STOP(Sect) \
typedef struct InstrProfValueData {
   /* Profiled value. */
   uint64_t Value;
   /* Number of times the value appears in the training run. */
   uint64_t Count;
 } InstrProfValueData;
 
 #endif /* INSTR_PROF_DATA_INC */
 
+#ifndef INSTR_ORDER_FILE_INC
+// The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB).
+#define INSTR_ORDER_FILE_BUFFER_SIZE 131072
+#define INSTR_ORDER_FILE_BUFFER_BITS 17
+#define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff
+#endif /* INSTR_ORDER_FILE_INC */
 #else
 #undef INSTR_PROF_DATA_DEFINED
 #endif

lib/profile/InstrProfiling.h

 const __llvm_profile_data *__llvm_profile_begin_data(void);
 const __llvm_profile_data *__llvm_profile_end_data(void);
 const char *__llvm_profile_begin_names(void);
 const char *__llvm_profile_end_names(void);
 uint64_t *__llvm_profile_begin_counters(void);
 uint64_t *__llvm_profile_end_counters(void);
 ValueProfNode *__llvm_profile_begin_vnodes();
 ValueProfNode *__llvm_profile_end_vnodes();
+uint32_t *__llvm_profile_begin_orderfile();
 
 /*!
  * \brief Clear profile counters to zero.
  *
  */
 void __llvm_profile_reset_counters(void);
 
 /*!

  * Writes to the file with the last name given to \a *
  * __llvm_profile_set_filename(),
  * or if it hasn't been called, the \c LLVM_PROFILE_FILE environment variable,
  * or if that's not set, the last name set to INSTR_PROF_PROFILE_NAME_VAR,
  * or if that's not set,  \c "default.profraw".
  */
 int __llvm_profile_write_file(void);
 
+int __llvm_orderfile_write_file(void);
 /*!
  * \brief this is a wrapper interface to \c __llvm_profile_write_file.
  * After this interface is invoked, a arleady dumped flag will be set
  * so that profile won't be dumped again during program exit. 
  * Invocation of interface __llvm_profile_reset_counters will clear
  * the flag. This interface is designed to be used to collect profile
  * data from user selected hot regions. The use model is
  *      __llvm_profile_reset_counters();
  *      ... hot region 1
  *      __llvm_profile_dump();
  *      .. some other code
  *      __llvm_profile_reset_counters();
  *       ... hot region 2
  *      __llvm_profile_dump();
  *
  *  It is expected that on-line profile merging is on with \c %m specifier
  *  used in profile filename . If merging is  not turned on, user is expected
  *  to invoke __llvm_profile_set_filename  to specify different profile names
  *  for different regions before dumping to avoid profile write clobbering.
  */
 int __llvm_profile_dump(void);
 
+int __llvm_orderfile_dump(void);
+
 /*!
  * \brief Set the filename for writing instrumentation data.
  *
  * Sets the filename to be used for subsequent calls to
  * \a __llvm_profile_write_file().
  *
  * \c Name is not copied, so it must remain valid.  Passing NULL resets the
  * filename logic to the default behaviour.

lib/profile/InstrProfilingFile.c

for (I = 0; I < NumIOVecs; I++) {
     } else {
       if (fseek(File, IOVecs[I].ElmSize * IOVecs[I].NumElm, SEEK_CUR) == -1)
         return 1;
     }
   }
   return 0;
 }
 
+/* TODO: make buffer size controllable by an internal option, and compiler can pass the size
+   to runtime via a variable. */
+static uint32_t orderFileWriter(FILE *File, const uint32_t *DataStart) {
+  if (fwrite(DataStart, sizeof(uint32_t), INSTR_ORDER_FILE_BUFFER_SIZE, File) !=
+      INSTR_ORDER_FILE_BUFFER_SIZE)
+    return 1;
+  return 0;
+}
+
 static void initFileWriter(ProfDataWriter *This, FILE *File) {
   This->Write = fileWriter;
   This->WriterCtx = File;
 }
 
 COMPILER_RT_VISIBILITY ProfBufferIO *
 lprofCreateBufferIOInternal(void *File, uint32_t BufferSz) {
   FreeHook = &free;
static int writeFile(const char *OutputName) {
   ProfDataWriter fileWriter;
   initFileWriter(&fileWriter, OutputFile);
   RetVal = lprofWriteData(&fileWriter, lprofGetVPDataReader(), MergeDone);
 
   fclose(OutputFile);
   return RetVal;
 }
 
+/* Write order data to file \c OutputName.  */
+static int writeOrderFile(const char *OutputName) {
+  int RetVal;
+  FILE *OutputFile;
+
+  OutputFile = fopen(OutputName, "w");
+
+  if (!OutputFile) {
+    PROF_WARN("can't open file with mode ab: %s\n", OutputName);
+    return -1;
+  }
+
+  FreeHook = &free;
+  setupIOBuffer();
+  const uint32_t *DataBegin = __llvm_profile_begin_orderfile();
+  RetVal = orderFileWriter(OutputFile, DataBegin);
+
+  fclose(OutputFile);
+  return RetVal;
+}
+
 static void truncateCurrentFile(void) {
   const char *Filename;
   char *FilenameBuf;
   FILE *File;
   int Length;
 
   Length = getCurFilenameLength();
   FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
PROF_WARN("Later invocation of __llvm_profile_dump can lead to clobbering "
               " of previously dumped profile data : %s. Either use %%m "
               "in profile name or change profile name before dumping.\n",
               "online profile merging is not on");
   int rc = __llvm_profile_write_file();
   lprofSetProfileDumped();
   return rc;
 }
 
+/* Order file data will be saved in a file with suffx .order. */
+static const char *OrderFileSuffix = ".order";
+
+COMPILER_RT_VISIBILITY
+int __llvm_orderfile_write_file(void) {
+  int rc, Length, LengthBeforeAppend, SuffixLength;
+  const char *Filename;
+  char *FilenameBuf;
+  int PDeathSig = 0;
+
+  SuffixLength = strlen(OrderFileSuffix);
+  Length = getCurFilenameLength() + SuffixLength;
+  FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
+  Filename = getCurFilename(FilenameBuf, 1);
+
+  /* Check the filename. */
+  if (!Filename) {
+    PROF_ERR("Failed to write file : %s\n", "Filename not set");
+    return -1;
+  }
+
+  /* Append order file suffix */
+  LengthBeforeAppend = strlen(Filename);
+  memcpy(FilenameBuf + LengthBeforeAppend, OrderFileSuffix, SuffixLength);
+  FilenameBuf[LengthBeforeAppend + SuffixLength] = '\0';
+
+  /* Check if there is llvm/runtime version mismatch.  */
+  if (GET_VERSION(__llvm_profile_get_version()) != INSTR_PROF_RAW_VERSION) {
+    PROF_ERR("Runtime and instrumentation version mismatch : "
+             "expected %d, but get %d\n",
+             INSTR_PROF_RAW_VERSION,
+             (int)GET_VERSION(__llvm_profile_get_version()));
+    return -1;
+  }
+
+  // Temporarily suspend getting SIGKILL when the parent exits.
+  PDeathSig = lprofSuspendSigKill();
+
+  /* Write order data to the file. */
+  rc = writeOrderFile(Filename);
+  if (rc)
+    PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno));
+
+  // Restore SIGKILL.
+  if (PDeathSig == 1)
+    lprofRestoreSigKill();
+
+  return rc;
+}
+
+COMPILER_RT_VISIBILITY
+int __llvm_orderfile_dump(void) {
+  int rc = __llvm_orderfile_write_file();
+  return rc;
+}
+
 static void writeFileWithoutReturn(void) { __llvm_profile_write_file(); }
 
 COMPILER_RT_VISIBILITY
 int __llvm_profile_register_write_file_atexit(void) {
   static int HasBeenRegistered = 0;
 
   if (HasBeenRegistered)
     return 0;
 
   lprofSetupValueProfiler();
 
   HasBeenRegistered = 1;
   return atexit(writeFileWithoutReturn);
 }
 
 #endif

lib/profile/InstrProfilingPlatformDarwin.c

 COMPILER_RT_VISIBILITY
 extern char NamesEnd __asm("section$end$__DATA$" INSTR_PROF_NAME_SECT_NAME);
 COMPILER_RT_VISIBILITY
 extern uint64_t
     CountersStart __asm("section$start$__DATA$" INSTR_PROF_CNTS_SECT_NAME);
 COMPILER_RT_VISIBILITY
 extern uint64_t
     CountersEnd __asm("section$end$__DATA$" INSTR_PROF_CNTS_SECT_NAME);
+COMPILER_RT_VISIBILITY
+extern uint32_t
+    OrderFileStart __asm("section$start$__DATA$" INSTR_PROF_ORDERFILE_SECT_NAME);
 
 COMPILER_RT_VISIBILITY
 extern ValueProfNode
     VNodesStart __asm("section$start$__DATA$" INSTR_PROF_VNODES_SECT_NAME);
 COMPILER_RT_VISIBILITY
 extern ValueProfNode
     VNodesEnd __asm("section$end$__DATA$" INSTR_PROF_VNODES_SECT_NAME);
 
 COMPILER_RT_VISIBILITY
 const __llvm_profile_data *__llvm_profile_begin_data(void) {
   return &DataStart;
 }
 COMPILER_RT_VISIBILITY
 const __llvm_profile_data *__llvm_profile_end_data(void) { return &DataEnd; }
 COMPILER_RT_VISIBILITY
 const char *__llvm_profile_begin_names(void) { return &NamesStart; }
 COMPILER_RT_VISIBILITY
 const char *__llvm_profile_end_names(void) { return &NamesEnd; }
 COMPILER_RT_VISIBILITY
 uint64_t *__llvm_profile_begin_counters(void) { return &CountersStart; }
 COMPILER_RT_VISIBILITY
 uint64_t *__llvm_profile_end_counters(void) { return &CountersEnd; }
+COMPILER_RT_VISIBILITY
+uint32_t *__llvm_profile_begin_orderfile(void) { return &OrderFileStart; }
 
 COMPILER_RT_VISIBILITY
 ValueProfNode *__llvm_profile_begin_vnodes(void) {
   return &VNodesStart;
 }
 COMPILER_RT_VISIBILITY
 ValueProfNode *__llvm_profile_end_vnodes(void) { return &VNodesEnd; }
 
 COMPILER_RT_VISIBILITY ValueProfNode *CurrentVNode = &VNodesStart;
 COMPILER_RT_VISIBILITY ValueProfNode *EndVNode = &VNodesEnd;
 #endif

test/profile/Inputs/instrprof-order-file-2.c

+__attribute__((noinline)) int f(int a) {
+  return a + 1;
+}
+
+__attribute__((noinline)) int g(int a) {
+  return a + 2;
+}

test/profile/Inputs/instrprof-order-file.c

+void __llvm_profile_initialize_file(void);
+int __llvm_orderfile_dump(void);
+
+__attribute__((noinline)) int f(int a);
+
+__attribute__((noinline)) int g(int a);
+
+int main(int argc, const char *argv[]) {
+  int a = f(argc);
+  int t = 0;
+  for (int i = 0; i < argc; i++)
+    t += g(a);
+  f(t);
+  __llvm_profile_initialize_file();
+  __llvm_orderfile_dump();
+  return 0;
+}

test/profile/instrprof-order-file.test

+// UNSUPPORTED: windows
+// RUN: rm -rf %t.dir && mkdir -p %t.dir
+// RUN: cd %t.dir
+//
+// RUN: %clang -forder-file-instrumentation -O1 -o %t.2 %S/Inputs/instrprof-order-file-2.c %S/Inputs/instrprof-order-file.c -mllvm -orderfile-write-mapping="mapping.txt"
+// RUN: %run %t.2 ANY
+// RUN: od -h default.profraw.order | FileCheck %s
+// RUN: cat mapping.txt | FileCheck %s --check-prefix=MAPPING
+
+// Make sure we have MD5 for main, then f, then g.
+// CHECK: 0000000      d5fa    e78d    6436    db95    a18f    dd4c    4f75    cc91
+// CHECK: 0000020      f5b2    47ff    6643    b671    0000    0000    0000    0000
+
+// MAPPING: MD5 cc914f75dd4ca18f f
+// MAPPING: MD5 b671664347fff5b2 g
+// MAPPING: MD5 db956436e78dd5fa main