Reset initial affinity in children processes

[jlpeyton]

Summary

If OpenMP is initialized before fork()-ing occurs and affinity is set to
something like compact, then the master thread will be pinned to a single HW
thread/core after initialization. If the master (or any other thread) then
forks N processes, all N processes will then be pinned to that same single HW
thread/core. To reset the affinity for the new child process, the atfork
handler for the child process can call kmp_set_thread_affinity_mask_initial() to
reset its affinity to the initial affinity of the application before it
re-initializes libomp. The parent process will not be affected and still
keeps its affinity setting.

[tlwilmar]

LGTM

openmp/trunk/runtime/src/kmp.h

 extern void __kmp_affinity_determine_capable(const char *env_var);
 extern int __kmp_aux_set_affinity(void **mask);
 extern int __kmp_aux_get_affinity(void **mask);
 extern int __kmp_aux_get_affinity_max_proc();
 extern int __kmp_aux_set_affinity_mask_proc(int proc, void **mask);
 extern int __kmp_aux_unset_affinity_mask_proc(int proc, void **mask);
 extern int __kmp_aux_get_affinity_mask_proc(int proc, void **mask);
 extern void __kmp_balanced_affinity(int tid, int team_size);
+#if KMP_OS_LINUX
+extern int kmp_set_thread_affinity_mask_initial(void);
+#endif
 #endif /* KMP_AFFINITY_SUPPORTED */
 
 extern void __kmp_cleanup_hierarchy();
 extern void __kmp_get_hierarchy(kmp_uint32 nproc, kmp_bstate_t *thr_bar);
 
 #if KMP_USE_FUTEX
 
 extern int __kmp_futex_determine_capable(void);

openmp/trunk/runtime/src/z_Linux_util.cpp

/* Reset the library so execution in the child starts "all over again" with
    allocated by parent, just abandon it to be safe. */
 static void __kmp_atfork_child(void) {
   /* TODO make sure this is done right for nested/sibling */
   // ATT:  Memory leaks are here? TODO: Check it and fix.
   /* KMP_ASSERT( 0 ); */
 
   ++__kmp_fork_count;
 
+#if KMP_AFFINITY_SUPPORTED && KMP_OS_LINUX
+  // reset the affinity in the child to the initial thread
+  // affinity in the parent
+  kmp_set_thread_affinity_mask_initial();
+#endif
+
   __kmp_init_runtime = FALSE;
 #if KMP_USE_MONITOR
   __kmp_init_monitor = 0;
 #endif
   __kmp_init_parallel = FALSE;
   __kmp_init_middle = FALSE;
   __kmp_init_serial = FALSE;
   TCW_4(__kmp_init_gtid, FALSE);