[Profile] Avoid deadlock in multithreaded context when forking with coverage enabled

[calixte]

Summary

When a program is forked with code coverage enabled, all the counters are flushed.
To avoid data race, a critical section has been added around __gcov_flush (https://reviews.llvm.org/D70910) but when forking the mutex is just duplicated and if it was in a locked state (in the parent process) then it's locked in the child process which leads to a deadlock when the child is trying to flush counters.
This patch aims to initialize the mutex in the child process when forking.
So a custom fork (__gcov_fork) is called from instrumented code and the mutex is then initialized in the child process.

clang/lib/Driver/ToolChains/Darwin.cpp

void Darwin::addProfileRTLibs(const ArgList &Args,
 
   bool ForGCOV = needsGCovInstrumentation(Args);
 
   // If we have a symbol export directive and we're linking in the profile
   // runtime, automatically export symbols necessary to implement some of the
   // runtime's functionality.
   if (hasExportSymbolDirective(Args)) {
     if (ForGCOV) {
+      addExportedSymbol(CmdArgs, "___gcov_fork");
       addExportedSymbol(CmdArgs, "___gcov_flush");
       addExportedSymbol(CmdArgs, "_flush_fn_list");
       addExportedSymbol(CmdArgs, "_writeout_fn_list");
     } else {
       addExportedSymbol(CmdArgs, "___llvm_profile_filename");
       addExportedSymbol(CmdArgs, "___llvm_profile_raw_version");
     }
     addExportedSymbol(CmdArgs, "_lprofDirMode");

compiler-rt/lib/profile/GCDAProfiling.c

 #include <pthread.h>
 static pthread_mutex_t gcov_flush_mutex = PTHREAD_MUTEX_INITIALIZER;
 static __inline void gcov_flush_lock() {
   pthread_mutex_lock(&gcov_flush_mutex);
 }
 static __inline void gcov_flush_unlock() {
   pthread_mutex_unlock(&gcov_flush_mutex);
 }
+static __inline void gcov_flush_init_mutex() {
+  pthread_mutex_init(&gcov_flush_mutex, NULL);
+}
 #else
 #include <windows.h>
 static SRWLOCK gcov_flush_mutex = SRWLOCK_INIT;
 static __inline void gcov_flush_lock() {
   AcquireSRWLockExclusive(&gcov_flush_mutex);
 }
 static __inline void gcov_flush_unlock() {
   ReleaseSRWLockExclusive(&gcov_flush_mutex);
 }
+static __inline void gcov_flush_init_mutex() {
+  InitializeSRWLock(&gcov_flush_mutex);
+}
 #endif
 
 /* #define DEBUG_GCDAPROFILING */
 /*
  * --- GCOV file format I/O primitives ---
  */
 
 /*
void llvm_delete_writeout_function_list(void) {
   fn_list_remove(&writeout_fn_list);
 }
 
 COMPILER_RT_VISIBILITY
 void llvm_register_flush_function(fn_ptr fn) {
   fn_list_insert(&flush_fn_list, fn);
 }
 
+pid_t __gcov_fork() {
+  pid_t pid = fork();
+
+  if (pid == 0) {
+    gcov_flush_init_mutex();
+  }
+
+  return pid;
+}
+
 void __gcov_flush() {
   gcov_flush_lock();
 
   struct fn_node* curr = flush_fn_list.head;
 
   while (curr) {
     curr->fn();
     curr = curr->next;

llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp

static bool shouldKeepInEntry(BasicBlock::iterator It) {
   if (auto *II = dyn_cast<IntrinsicInst>(It)) {
     if (II->getIntrinsicID() == llvm::Intrinsic::localescape) return true;
   }
 
   return false;
 }
 
 void GCOVProfiler::AddFlushBeforeForkAndExec() {
-  SmallVector<Instruction *, 2> ForkAndExecs;
+  SmallVector<std::pair<bool, CallInst *>, 2> ForksAndExecs;
   for (auto &F : M->functions()) {
     auto *TLI = &GetTLI(F);
     for (auto &I : instructions(F)) {
       if (CallInst *CI = dyn_cast<CallInst>(&I)) {
         if (Function *Callee = CI->getCalledFunction()) {
           LibFunc LF;
-          if (TLI->getLibFunc(*Callee, LF) &&
-              (LF == LibFunc_fork || LF == LibFunc_execl ||
-               LF == LibFunc_execle || LF == LibFunc_execlp ||
-               LF == LibFunc_execv || LF == LibFunc_execvp ||
-               LF == LibFunc_execve || LF == LibFunc_execvpe ||
-               LF == LibFunc_execvP)) {
-            ForkAndExecs.push_back(&I);
+          if (TLI->getLibFunc(*Callee, LF)) {
+            if (LF == LibFunc_fork) {
+              ForksAndExecs.push_back({true, CI});
+            } else if (LF == LibFunc_execl || LF == LibFunc_execle ||
+                       LF == LibFunc_execlp || LF == LibFunc_execv ||
+                       LF == LibFunc_execvp || LF == LibFunc_execve ||
+                       LF == LibFunc_execvpe || LF == LibFunc_execvP) {
+              ForksAndExecs.push_back({false, CI});
+            }
           }
         }
       }
     }
   }
 
   // We need to split the block after the fork/exec call
   // because else the counters for the lines after will be
   // the same as before the call.
-  for (auto I : ForkAndExecs) {
-    IRBuilder<> Builder(I);
+  for (auto F : ForksAndExecs) {
+    IRBuilder<> Builder(F.second);
     FunctionType *FTy = FunctionType::get(Builder.getVoidTy(), {}, false);
     FunctionCallee GCOVFlush = M->getOrInsertFunction("__gcov_flush", FTy);
     Builder.CreateCall(GCOVFlush);
-    I->getParent()->splitBasicBlock(I);
+
+    if (F.first) {
+      // We've a fork, so we call a custom fork which will initialize the mutex
+      // used when flushing
+      FTy = FunctionType::get(Builder.getInt32Ty(), {}, false);
+      FunctionCallee GCOVFork = M->getOrInsertFunction("__gcov_fork", FTy);
+      F.second->setCalledFunction(GCOVFork);
+    }
+
+    F.second->getParent()->splitBasicBlock(F.second);
   }
 }
 
 void GCOVProfiler::emitProfileNotes() {
   NamedMDNode *CU_Nodes = M->getNamedMetadata("llvm.dbg.cu");
   if (!CU_Nodes) return;
 
   for (unsigned i = 0, e = CU_Nodes->getNumOperands(); i != e; ++i) {