[PGO] Suspend SIGKILL for PR_SET_PDEATHSIG in profile-write

[xur]

Summary

We found a nondeterministic behavior when doing online profile merging
for multi-process applications. The application forks a sub-process and
sub-process sets to get SIGKILL when the parent process exits,

The first process gets the lock, and dumps the profile. The second one
will mmap the file, do the merge and write out the file. Note that before
the merged write, we truncate the profile.

Depending on the timing, the child process might be terminated
abnormally when the parent exits first. If this happens:
(1) before the truncation, we will get the profile for the main process
(2) after the truncation, and before write-out the profile, we will get
0 size profile.
(3) after the merged write, we get merged profile.

This patch temporarily suspend the SIGKILL for PR_SET_PDEATHSIG
before profile-write and restore it after the write.

This patch only applies to Linux system.

[davidxl]

lib/profile/InstrProfilingFile.c

556

Put the platform dependent utility into InstProfilingUtil.c file.

void lprofSuspendSigKill(int *pdeathsig) {
#ifdef __linux__
...
#else 
#endif
}

void lprofRestoreSigKill(int pdeathsig) {
..
}

[xur]

move platform depended code to InstrProfilingUnil.c per David's suggestion.

[davidxl]

Can you create a test case? It can be put under the Linux subdir.

[xur]

Add a testcase

sing a fake count section to increase the profile size.
Main process gets to profile run-time first and dump the profile.
It will then exit when child process writes the profile.
Without this patch, there is a high chance of getting 0 size profile.

[davidxl]

test/profile/Linux/prctl.c

18

format the test in llvm style.

24

50 us or microsec

34

10us

41

Just check size of profile data? It is not guaranteed that child process will have a chance to dump.

[xur]

Integrated David's review comments.

An addition change is the adding of 'sleep 5' b/w run and llvm-profdata
because if we run llvm-profdata immediately after. It's possible to get a
0-size files as it's being truncated by the child process (ie. the main
process exit while the child process is still accessing the file).

[davidxl]

lgtm

test/profile/Linux/prctl.c

3

sleep 5 seconds can be a problem -- it greatly increases the total runtime of check-profile. Please tune this down to 1 second (by reducing the fake data size).

lib/profile/InstrProfilingFile.c

  * set by previous calls to __llvm_profile_set_filename or
  * __llvm_profile_override_default_filename or
  * __llvm_profile_initialize_file. */
 COMPILER_RT_VISIBILITY
 int __llvm_profile_write_file(void) {
   int rc, Length;
   const char *Filename;
   char *FilenameBuf;
+  int PDeathSig = 0;
 
   if (lprofProfileDumped()) {
     PROF_NOTE("Profile data not written to file: %s.\n", 
               "already written");
     return 0;
   }
 
   Length = getCurFilenameLength();
int __llvm_profile_write_file(void) {
   if (GET_VERSION(__llvm_profile_get_version()) != INSTR_PROF_RAW_VERSION) {
     PROF_ERR("Runtime and instrumentation version mismatch : "
              "expected %d, but get %d\n",
              INSTR_PROF_RAW_VERSION,
              (int)GET_VERSION(__llvm_profile_get_version()));
     return -1;
   }
 
+  // Temporarily suspend getting SIGKILL when the parent exits.
+  PDeathSig = lprofSuspendSigKill();
+
   /* Write profile data to the file. */
   rc = writeFile(Filename);
   if (rc)
     PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno));
+
+  // Restore SIGKILL.
+  if (PDeathSig == 1)
+    lprofRestoreSigKill();
+
   return rc;
 }
 
 COMPILER_RT_VISIBILITY
 int __llvm_profile_dump(void) {
   if (!doMerging())
     PROF_WARN("Later invocation of __llvm_profile_dump can lead to clobbering "
               " of previously dumped profile data : %s. Either use %%m "

lib/profile/InstrProfilingUtil.h

  * the string \c Path, or NULL if the char is not found. */
 const char *lprofFindLastDirSeparator(const char *Path);
 
 int lprofGetHostName(char *Name, int Len);
 
 unsigned lprofBoolCmpXchg(void **Ptr, void *OldV, void *NewV);
 void *lprofPtrFetchAdd(void **Mem, long ByteIncr);
 
+/* Temporarily suspend SIGKILL. Return value of 1 means a restore is needed.
+ * Other return values mean no restore is needed.
+ */
+int lprofSuspendSigKill();
+
+/* Restore previously suspended SIGKILL. */
+void lprofRestoreSigKill();
+
 #endif /* PROFILE_INSTRPROFILINGUTIL_H */

lib/profile/InstrProfilingUtil.c

 
 #ifdef COMPILER_RT_HAS_UNAME
 #include <sys/utsname.h>
 #endif
 
 #include <stdlib.h>
 #include <string.h>
 
+#if defined(__linux__)
+#include <signal.h>
+#include <sys/prctl.h>
+#endif
+
 COMPILER_RT_VISIBILITY
 void __llvm_profile_recursive_mkdir(char *path) {
   int i;
 
   for (i = 1; path[i] != '\0'; ++i) {
     char save = path[i];
     if (!IS_DIR_SEPARATOR(path[i]))
       continue;
COMPILER_RT_VISIBILITY const char *lprofFindLastDirSeparator(const char *Path) {
   Sep = strrchr(Path, DIR_SEPARATOR);
   if (Sep)
     return Sep;
 #if defined(DIR_SEPARATOR_2)
   Sep = strrchr(Path, DIR_SEPARATOR_2);
 #endif
   return Sep;
 }
+
+COMPILER_RT_VISIBILITY int lprofSuspendSigKill() {
+#if defined(__linux__)
+  int PDeachSig = 0;
+  /* Temporarily suspend getting SIGKILL upon exit of the parent process. */
+  if (prctl(PR_GET_PDEATHSIG, &PDeachSig) == 0 && PDeachSig == SIGKILL) {
+    fprintf(stderr, "set\n");
+    prctl(PR_SET_PDEATHSIG, 0);
+  }
+  return (PDeachSig == SIGKILL);
+#else
+  return 0;
+#endif
+}
+
+COMPILER_RT_VISIBILITY void lprofRestoreSigKill() {
+#if defined(__linux__)
+  fprintf(stderr, "restore \n");
+  prctl(PR_SET_PDEATHSIG, SIGKILL);
+#endif
+}

test/profile/Linux/prctl.c

+// RUN: %clang_pgogen -O2 -o %t %s
+// RUN: rm -rf default_*.profraw
+// RUN: %run %t && sleep 5
+// RUN: llvm-profdata show default_*.profraw 2>&1 | FileCheck %s
+
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/prctl.h>
+#include <unistd.h>
+
+#define FAKE_COUNT_SZ 10000000
+/* fake counts to increse the profile size. */
+unsigned long long __attribute__((section("__llvm_prf_cnts")))
+counts[FAKE_COUNT_SZ];
+
+int main(int argc, char **argv) {
+  pid_t pid = fork();
+  if (pid == 0) {
+    int i;
+    int sum = 0;
+    /* child process: sleep 500us and get to runtime before the
+     * main process exits. */
+    prctl(PR_SET_PDEATHSIG, SIGKILL);
+    usleep(500);
+    for (i = 0; i < 5000; ++i)
+      sum += i * i * i;
+    printf("child process (%d): sum=%d\n", getpid(), sum);
+  } else if (pid > 0) {
+    /* parent process: sleep 100us to get into profile runtime first. */
+    usleep(100);
+  }
+  return 0;
+}
+
+// CHECK-NOT: Empty raw profile file