Improve stability of the runtime in parent/child processes

[hbae]

Summary

This change improves stability of the runtime when the application forks child processes.
Acquiring/releasing kmp_initz_lock and kmp_forkjoin_lock in the atfork handlers insures that the actual fork does not occur while those two locks are held, and __kmp_itt_reset() reverts the itt's global state to the initial state which also initializes the mutex stored in the global state.
Some missing initialization code was also inserted in the child's atfork handler.

[AndreyChurbanov]

LGTM

runtime/src/kmp.h

 extern char const *__kmp_barrier_branch_bit_env_name[bs_last_barrier];
 extern char const *__kmp_barrier_pattern_env_name[bs_last_barrier];
 extern char const *__kmp_barrier_type_name[bs_last_barrier];
 extern char const *__kmp_barrier_pattern_name[bp_last_bar];
 
 /* Global Locks */
 extern kmp_bootstrap_lock_t __kmp_initz_lock; /* control initialization */
 extern kmp_bootstrap_lock_t __kmp_forkjoin_lock; /* control fork/join access */
+extern kmp_bootstrap_lock_t __kmp_task_team_lock;
 extern kmp_bootstrap_lock_t
     __kmp_exit_lock; /* exit() is not always thread-safe */
 #if KMP_USE_MONITOR
 extern kmp_bootstrap_lock_t
     __kmp_monitor_lock; /* control monitor thread creation */
 #endif
 extern kmp_bootstrap_lock_t
     __kmp_tp_cached_lock; /* used for the hack to allow threadprivate cache and

 /* ------------------------------------------------------------------------- */
 
 /* the following are protected by the fork/join lock */
 /* write: lock  read: anytime */
 extern kmp_info_t **__kmp_threads; /* Descriptors for the threads */
 /* read/write: lock */
 extern volatile kmp_team_t *__kmp_team_pool;
 extern volatile kmp_info_t *__kmp_thread_pool;
+extern kmp_info_t *__kmp_thread_pool_insert_pt;
 
 // total num threads reachable from some root thread including all root threads
 extern volatile int __kmp_nth;
 /* total number of threads reachable from some root thread including all root
    threads, and those in the thread pool */
 extern volatile int __kmp_all_nth;
 extern int __kmp_thread_pool_nth;
 extern volatile int __kmp_thread_pool_active_nth;

runtime/src/kmp_itt.h

 #endif
 
 // Simplify the handling of an argument that is only required when USE_ITT_BUILD
 // is enabled.
 #define USE_ITT_BUILD_ARG(x) , x
 
 void __kmp_itt_initialize();
 void __kmp_itt_destroy();
+void __kmp_itt_reset();
 
 // -----------------------------------------------------------------------------
 // New stuff for reporting high-level constructs.
 
 // Note the naming convention:
 //     __kmp_itt_xxxing() function should be called before action, while
 //     __kmp_itt_xxxed()  function should be called after action.
 

runtime/src/kmp_itt.cpp

 #include "kmp_itt.h"
 
 #if KMP_DEBUG
 #include "kmp_itt.inl"
 #endif
 
 #if USE_ITT_NOTIFY
 
+#include "ittnotify_config.h"
+__itt_global __kmp_ittapi_clean_global;
+extern __itt_global __kmp_itt__ittapi_global;
 kmp_int32 __kmp_barrier_domain_count;
 kmp_int32 __kmp_region_domain_count;
 __itt_domain *__kmp_itt_barrier_domains[KMP_MAX_FRAME_DOMAINS];
 __itt_domain *__kmp_itt_region_domains[KMP_MAX_FRAME_DOMAINS];
 __itt_domain *__kmp_itt_imbalance_domains[KMP_MAX_FRAME_DOMAINS];
 kmp_int32 __kmp_itt_region_team_size[KMP_MAX_FRAME_DOMAINS];
 __itt_domain *metadata_domain = NULL;
 __itt_string_handle *string_handle_imbl = NULL;

 */
 
 kmp_int32 __kmp_itt_prepare_delay = 0;
 kmp_bootstrap_lock_t __kmp_itt_debug_lock =
     KMP_BOOTSTRAP_LOCK_INITIALIZER(__kmp_itt_debug_lock);
 
 #endif // USE_ITT_NOTIFY
 
+void __kmp_itt_reset() {
+#if USE_ITT_NOTIFY
+  __kmp_itt__ittapi_global = __kmp_ittapi_clean_global;
+#endif
+}
+
 void __kmp_itt_initialize() {
 
 // ITTNotify library is loaded and initialized at first call to any ittnotify
 // function, so we do not need to explicitly load it any more. Just report OMP
 // RTL version to ITTNotify.
 
 #if USE_ITT_NOTIFY
+  // Backup a clean global state
+  __kmp_ittapi_clean_global = __kmp_itt__ittapi_global;
+
   // Report OpenMP RTL version.
   kmp_str_buf_t buf;
   __itt_mark_type version;
   __kmp_str_buf_init(&buf);
   __kmp_str_buf_print(&buf, "OMP RTL Version %d.%d.%d", __kmp_version_major,
                       __kmp_version_minor, __kmp_version_build);
   if (__itt_api_version_ptr != NULL) {
     __kmp_str_buf_print(&buf, ":%s", __itt_api_version());

runtime/src/kmp_runtime.cpp

 #endif
 
 static int __kmp_expand_threads(int nWish, int nNeed);
 #if KMP_OS_WINDOWS
 static int __kmp_unregister_root_other_thread(int gtid);
 #endif
 static void __kmp_unregister_library(void); // called by __kmp_internal_end()
 static void __kmp_reap_thread(kmp_info_t *thread, int is_root);
-static kmp_info_t *__kmp_thread_pool_insert_pt = NULL;
+kmp_info_t *__kmp_thread_pool_insert_pt = NULL;
 
 /* Calculate the identifier of the current thread */
 /* fast (and somewhat portable) way to get unique identifier of executing
    thread. Returns KMP_GTID_DNE if we haven't been assigned a gtid. */
 int __kmp_get_global_thread_id() {
   int i;
   kmp_info_t **other_threads;
   size_t stack_data;

runtime/src/kmp_tasking.cpp

  * useful outside the context of tasking, then this will have to change, but
  * not settting the field minimizes the performance impact of tasking on
  * barriers, when no explicit tasks were spawned (pushed, actually).
  */
 
 static kmp_task_team_t *__kmp_free_task_teams =
     NULL; // Free list for task_team data structures
 // Lock for task team data structures
-static kmp_bootstrap_lock_t __kmp_task_team_lock =
+kmp_bootstrap_lock_t __kmp_task_team_lock =
     KMP_BOOTSTRAP_LOCK_INITIALIZER(__kmp_task_team_lock);
 
 // __kmp_alloc_task_deque:
 // Allocates a task deque for a particular thread, and initialize the necessary
 // data structures relating to the deque.  This only happens once per thread
 // per task team since task teams are recycled. No lock is needed during
 // allocation since each thread allocates its own deque.
 static void __kmp_alloc_task_deque(kmp_info_t *thread,

runtime/src/z_Linux_util.cpp

 void __kmp_disable(int *old_state) {
 #ifdef KMP_CANCEL_THREADS
   int status;
   status = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, old_state);
   KMP_CHECK_SYSFAIL("pthread_setcancelstate", status);
 #endif
 }
 
-static void __kmp_atfork_prepare(void) { /*  nothing to do  */
+static void __kmp_atfork_prepare(void) {
+  __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
+  __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
 }
 
-static void __kmp_atfork_parent(void) { /*  nothing to do  */
+static void __kmp_atfork_parent(void) {
+  __kmp_release_bootstrap_lock(&__kmp_initz_lock);
+  __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
 }
 
 /* Reset the library so execution in the child starts "all over again" with
    clean data structures in initial states.  Don't worry about freeing memory
    allocated by parent, just abandon it to be safe. */
 static void __kmp_atfork_child(void) {
+  __kmp_release_bootstrap_lock(&__kmp_initz_lock);
+  __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
   /* TODO make sure this is done right for nested/sibling */
   // ATT:  Memory leaks are here? TODO: Check it and fix.
   /* KMP_ASSERT( 0 ); */
 
   ++__kmp_fork_count;
 
 #if KMP_AFFINITY_SUPPORTED
 #if KMP_OS_LINUX
#if !KMP_USE_DYNAMIC_LOCK
   __kmp_user_lock_table.allocated = 0;
   __kmp_user_lock_table.table = NULL;
   __kmp_lock_blocks = NULL;
 #endif
 
   __kmp_all_nth = 0;
   TCW_4(__kmp_nth, 0);
 
+  __kmp_thread_pool = NULL;
+  __kmp_thread_pool_insert_pt = NULL;
+  __kmp_team_pool = NULL;
+
   /* Must actually zero all the *cache arguments passed to __kmpc_threadprivate
      here so threadprivate doesn't use stale data */
   KA_TRACE(10, ("__kmp_atfork_child: checking cache address list %p\n",
                 __kmp_threadpriv_cache_list));
 
   while (__kmp_threadpriv_cache_list != NULL) {
 
     if (*__kmp_threadpriv_cache_list->addr != NULL) {
       KC_TRACE(50, ("__kmp_atfork_child: zeroing cache at address %p\n",
                     &(*__kmp_threadpriv_cache_list->addr)));
 
       *__kmp_threadpriv_cache_list->addr = NULL;
     }
     __kmp_threadpriv_cache_list = __kmp_threadpriv_cache_list->next;
   }
 
   __kmp_init_runtime = FALSE;
 
   /* reset statically initialized locks */
   __kmp_init_bootstrap_lock(&__kmp_initz_lock);
   __kmp_init_bootstrap_lock(&__kmp_stdio_lock);
   __kmp_init_bootstrap_lock(&__kmp_console_lock);
+  __kmp_init_bootstrap_lock(&__kmp_task_team_lock);
+
+  __kmp_itt_reset(); // reset ITT's global state
 
   /* This is necessary to make sure no stale data is left around */
   /* AC: customers complain that we use unsafe routines in the atfork
      handler. Mathworks: dlsym() is unsafe. We call dlsym and dlopen
      in dynamic_link when check the presence of shared tbbmalloc library.
      Suggestion is to make the library initialization lazier, similar
      to what done for __kmpc_begin(). */
   // TODO: synchronize all static initializations with regular library