Fix for pthread_setspecific (TLS and shutdown) problem

[jlpeyton]

Summary

Some codes that use TLS fail intermittently because one thread tries to write TLS values after the TLS key has been destroyed by another thread. This happens when one thread executes library shutdown (and destroys TLS keys), while another thread starts to execute the TLS key destructor routine. Before this change, the kmp_init_runtime flag was checked before calling pthread_* TLS functions, but this flag is set to FALSE later than the destruction of the TLS keys, which leads to failure. The fix is to check kmp_init_gtid instead, as this flag is unset *before* the destruction of TLS keys.

[AndreyChurbanov]

LGTM

runtime/src/kmp_runtime.c

#endif
     root->r.r_root_team->t.t_threads[0] = root_thread;
     root->r.r_hot_team ->t.t_threads[0] = root_thread;
     root_thread->th.th_serial_team->t.t_threads[0] = root_thread;
     root_thread->th.th_serial_team->t.t_serialized = 0; // AC: the team created in reserve, not for execution (it is unused for now).
     root->r.r_uber_thread = root_thread;
 
     /* initialize the thread, get it ready to go */
     __kmp_initialize_info( root_thread, root->r.r_root_team, 0, gtid );
+    TCW_4(__kmp_init_gtid, TRUE);
 
     /* prepare the master thread for get_gtid() */
     __kmp_gtid_set_specific( gtid );
 
     __kmp_itt_thread_name( gtid );
 
     #ifdef KMP_TDATA_GTID
         __kmp_gtid = gtid;
     #endif
     __kmp_create_worker( gtid, root_thread, __kmp_stksize );
     KMP_DEBUG_ASSERT( __kmp_gtid_get_specific() == gtid );
-    TCW_4(__kmp_init_gtid, TRUE);
 
     KA_TRACE( 20, ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, plain=%u\n",
                     gtid, __kmp_gtid_from_tid( 0, root->r.r_hot_team ),
                     root->r.r_hot_team->t.t_id, 0, KMP_INIT_BARRIER_STATE,
                     KMP_INIT_BARRIER_STATE ) );
     { // Initialize barrier data.
         int b;
         for ( b = 0; b < bs_last_barrier; ++ b ) {

runtime/src/z_Linux_util.c

 }
 
 /* ------------------------------------------------------------------------ */
 /* ------------------------------------------------------------------------ */
 
 void
 __kmp_gtid_set_specific( int gtid )
 {
+    if( __kmp_init_gtid ) {
         int status;
-    KMP_ASSERT( __kmp_init_runtime );
         status = pthread_setspecific( __kmp_gtid_threadprivate_key, (void*)(intptr_t)(gtid+1) );
         KMP_CHECK_SYSFAIL( "pthread_setspecific", status );
+    } else {
+        KA_TRACE( 50, ("__kmp_gtid_set_specific: runtime shutdown, returning\n" ) );
+    }
 }
 
 int
 __kmp_gtid_get_specific()
 {
     int gtid;
-    if ( !__kmp_init_runtime ) {
-        KA_TRACE( 50, ("__kmp_get_specific: runtime shutdown, returning KMP_GTID_SHUTDOWN\n" ) );
+    if ( !__kmp_init_gtid ) {
+        KA_TRACE( 50, ("__kmp_gtid_get_specific: runtime shutdown, returning KMP_GTID_SHUTDOWN\n" ) );
         return KMP_GTID_SHUTDOWN;
     }
     gtid = (int)(size_t)pthread_getspecific( __kmp_gtid_threadprivate_key );
     if ( gtid == 0 ) {
         gtid = KMP_GTID_DNE;
     }
     else {
         gtid--;

runtime/src/z_Windows_NT_util.c

 }
 
 /* ------------------------------------------------------------------------ */
 /* ------------------------------------------------------------------------ */
 
 void
 __kmp_gtid_set_specific( int gtid )
 {
+    if( __kmp_init_gtid ) {
         KA_TRACE( 50, ("__kmp_gtid_set_specific: T#%d key:%d\n",
                     gtid, __kmp_gtid_threadprivate_key ));
-    KMP_ASSERT( __kmp_init_runtime );
         if( ! TlsSetValue( __kmp_gtid_threadprivate_key, (LPVOID)(gtid+1)) )
             KMP_FATAL( TLSSetValueFailed );
+    } else {
+        KA_TRACE( 50, ("__kmp_gtid_set_specific: runtime shutdown, returning\n" ) );
+    }
 }
 
 int
 __kmp_gtid_get_specific()
 {
     int gtid;
-    if( !__kmp_init_runtime ) {
-        KA_TRACE( 50, ("__kmp_get_specific: runtime shutdown, returning KMP_GTID_SHUTDOWN\n" ) );
+    if( !__kmp_init_gtid ) {
+        KA_TRACE( 50, ("__kmp_gtid_get_specific: runtime shutdown, returning KMP_GTID_SHUTDOWN\n" ) );
         return KMP_GTID_SHUTDOWN;
     }
     gtid = (int)(kmp_intptr_t)TlsGetValue( __kmp_gtid_threadprivate_key );
     if ( gtid == 0 ) {
         gtid = KMP_GTID_DNE;
     }
     else {
         gtid--;