Refactor of task_team code.

[jlpeyton]

Summary

This is a refactoring of the task_team code that more elegantly handles the two
task_team case. Two task_teams per team are kept in use for the lifetime of
the team. Thus no reference counting is needed.

[AndreyChurbanov]

LGTM

openmp/trunk/runtime/src/kmp.h

#if OMP_41_ENABLED
     kmp_int32               tt_found_proxy_tasks;  /* Have we found proxy tasks since last barrier */
 #endif
 
     KMP_ALIGN_CACHE
     volatile kmp_uint32     tt_unfinished_threads; /* #threads still active      */
 
     KMP_ALIGN_CACHE
     volatile kmp_uint32     tt_active;             /* is the team still actively executing tasks */
-
-    KMP_ALIGN_CACHE
-#if KMP_USE_INTERNODE_ALIGNMENT
-    kmp_int32               tt_padme[INTERNODE_CACHE_LINE/sizeof(kmp_int32)];
-#endif
-
-    volatile kmp_uint32     tt_ref_ct;             /* #threads accessing struct  */
-                                                   /* (not incl. master)         */
 } kmp_base_task_team_t;
 
 union KMP_ALIGN_CACHE kmp_task_team {
     kmp_base_task_team_t tt;
     double               tt_align;       /* use worst case alignment */
     char                 tt_pad[ KMP_PAD(kmp_base_task_team_t, CACHE_LINE) ];
 };
 
#endif /* USE_ITT_BUILD */
                            kmp_int32 is_constrained);
 int __kmp_execute_tasks_oncore(kmp_info_t *thread, kmp_int32 gtid, kmp_flag_oncore *flag, int final_spin,
                                int *thread_finished,
 #if USE_ITT_BUILD
                                void * itt_sync_obj,
 #endif /* USE_ITT_BUILD */
                                kmp_int32 is_constrained);
 
+extern void __kmp_free_task_team( kmp_info_t *thread, kmp_task_team_t *task_team );
 extern void __kmp_reap_task_teams( void );
-extern void __kmp_unref_task_team( kmp_task_team_t *task_team, kmp_info_t *thread );
 extern void __kmp_wait_to_unref_task_teams( void );
-extern void __kmp_task_team_setup ( kmp_info_t *this_thr, kmp_team_t *team, int both, int always );
+extern void __kmp_task_team_setup ( kmp_info_t *this_thr, kmp_team_t *team, int always );
 extern void __kmp_task_team_sync  ( kmp_info_t *this_thr, kmp_team_t *team );
 extern void __kmp_task_team_wait  ( kmp_info_t *this_thr, kmp_team_t *team
 #if USE_ITT_BUILD
                                     , void * itt_sync_obj
 #endif /* USE_ITT_BUILD */
+                                    , int wait=1
 );
 extern void __kmp_tasking_barrier( kmp_team_t *team, kmp_info_t *thread, int gtid );
 
 extern int  __kmp_is_address_mapped( void *addr );
 extern kmp_uint64 __kmp_hardware_timestamp(void);
 
 #if KMP_OS_UNIX
 extern int  __kmp_read_from_file( char const *path, char const *format, ... );

openmp/trunk/runtime/src/kmp_barrier.cpp

#endif /* USE_DEBUGGER */
 
         KMP_MB();
 
         if (KMP_MASTER_TID(tid)) {
             status = 0;
             if (__kmp_tasking_mode != tskm_immediate_exec) {
                 __kmp_task_team_wait(this_thr, team
                                      USE_ITT_BUILD_ARG(itt_sync_obj) );
-                __kmp_task_team_setup(this_thr, team, 0, 0); // use 0,0 to only setup the current team if nthreads > 1
+                __kmp_task_team_setup(this_thr, team, 0); // use 0 to only setup the current team if nthreads > 1
             }
 #if USE_DEBUGGER
             // Let the debugger know: All threads are arrived and starting leaving the barrier.
             team->t.t_bar[bt].b_team_arrived += 1;
 #endif
 
 #if USE_ITT_BUILD
             /* TODO: In case of split reduction barrier, master thread may send acquired event early,
#if USE_ITT_NOTIFY
                     itt_sync_obj = __kmp_itt_barrier_object(gtid, bt, 1);
                     __kmp_itt_barrier_starting(gtid, itt_sync_obj);
                 }
 #endif
 
                 KMP_DEBUG_ASSERT(this_thr->th.th_task_team->tt.tt_found_proxy_tasks == TRUE);
                 __kmp_task_team_wait(this_thr, team
                                                USE_ITT_BUILD_ARG(itt_sync_obj));
-                __kmp_task_team_setup(this_thr, team, 0, 0);
+                __kmp_task_team_setup(this_thr, team, 0);
 
 #if USE_ITT_BUILD
                 if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
                     __kmp_itt_barrier_finished(gtid, itt_sync_obj);
 #endif /* USE_ITT_BUILD */
             }
 #else
             // The task team should be NULL for serialized code (tasks will be executed immediately)
#ifdef KMP_DEBUG
             KMP_DEBUG_ASSERT((TCR_4(other_threads[i]->th.th_bar[bs_forkjoin_barrier].bb.b_go)
                               & ~(KMP_BARRIER_SLEEP_STATE))
                              == KMP_INIT_BARRIER_STATE);
             KMP_DEBUG_ASSERT(other_threads[i]->th.th_team == team);
         }
 #endif
 
         if (__kmp_tasking_mode != tskm_immediate_exec) {
-            __kmp_task_team_setup(this_thr, team, 1, 0);  // 1,0 indicates setup both task teams if nthreads > 1
+            __kmp_task_team_setup(this_thr, team, 0);  // 0 indicates setup current task team if nthreads > 1
         }
 
         /* The master thread may have changed its blocktime between the join barrier and the
            fork barrier. Copy the blocktime info to the thread, where __kmp_wait_template() can
            access it when the team struct is not guaranteed to exist. */
         // See note about the corresponding code in __kmp_join_barrier() being performance-critical
         if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
             this_thr->th.th_team_bt_intervals = team->t.t_implicit_task_taskdata[tid].td_icvs.bt_intervals;
#endif
     default: {
         __kmp_linear_barrier_release(bs_forkjoin_barrier, this_thr, gtid, tid, TRUE
                                      USE_ITT_BUILD_ARG(itt_sync_obj) );
     }
     }
 
     // Early exit for reaping threads releasing forkjoin barrier
     if (TCR_4(__kmp_global.g.g_done)) {
-        if (this_thr->th.th_task_team != NULL) {
-            if (KMP_MASTER_TID(tid)) {
-                TCW_PTR(this_thr->th.th_task_team, NULL);
-            }
-            else {
-                __kmp_unref_task_team(this_thr->th.th_task_team, this_thr);
-            }
-        }
+        this_thr->th.th_task_team = NULL;
 
 #if USE_ITT_BUILD && USE_ITT_NOTIFY
         if (__itt_sync_create_ptr || KMP_ITT_DEBUG) {
             if (!KMP_MASTER_TID(tid)) {
                 itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier);
                 if (itt_sync_obj)
                     __kmp_itt_barrier_finished(gtid, itt_sync_obj);
             }

openmp/trunk/runtime/src/kmp_runtime.c

 #endif
         KA_TRACE( 20, ( "__kmp_fork_call: Master T#%d pushing task_team %p / team %p, new task_team %p / team %p\n",
                       __kmp_gtid_from_thread( master_th ), master_th->th.th_task_team,
                       parent_team, team->t.t_task_team[master_th->th.th_task_state], team ) );
         if (level) {
             // Take a memo of master's task_state
             KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack);
             if (master_th->th.th_task_state_top >= master_th->th.th_task_state_stack_sz) { // increase size
-                kmp_uint8 *old_stack, *new_stack = (kmp_uint8 *) __kmp_allocate( 2*master_th->th.th_task_state_stack_sz );
+                kmp_uint32 new_size = 2*master_th->th.th_task_state_stack_sz;
+                kmp_uint8 *old_stack, *new_stack;
                 kmp_uint32 i;
+                new_stack = (kmp_uint8 *)__kmp_allocate(new_size);
                 for (i=0; i<master_th->th.th_task_state_stack_sz; ++i) {
                     new_stack[i] = master_th->th.th_task_state_memo_stack[i];
                 }
+                for (i=master_th->th.th_task_state_stack_sz; i<new_size; ++i) { // zero-init rest of stack
+                    new_stack[i] = 0;
+                }
                 old_stack = master_th->th.th_task_state_memo_stack;
                 master_th->th.th_task_state_memo_stack = new_stack;
-                master_th->th.th_task_state_stack_sz *= 2;
+                master_th->th.th_task_state_stack_sz = new_size;
                 __kmp_free(old_stack);
             }
             // Store master's task_state on stack
             master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top] = master_th->th.th_task_state;
             master_th->th.th_task_state_top++;
+            if (team == master_th->th.th_hot_teams[level].hot_team) { // Restore master's nested state if nested hot team
+                master_th->th.th_task_state = master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top];
+            }
+            else {
                 master_th->th.th_task_state = 0;
             }
-        master_th->th.th_task_team = team->t.t_task_team[master_th->th.th_task_state];
-
+        }
 #if !KMP_NESTED_HOT_TEAMS
         KMP_DEBUG_ASSERT((master_th->th.th_task_team == NULL) || (team == root->r.r_hot_team));
 #endif
     }
 
     KA_TRACE( 20, ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n",
                 gtid, parent_team->t.t_id, team->t.t_master_tid, team->t.t_id, team->t.t_nproc ));
     KMP_DEBUG_ASSERT( team != root->r.r_hot_team ||
if ( master_th->th.th_teams_microtask &&
         team->t.t_active_level --;
         KMP_TEST_THEN_DEC32( (kmp_int32*) &root->r.r_in_parallel );
 
         /* Restore number of threads in the team if needed */
         if ( master_th->th.th_team_nproc < master_th->th.th_teams_size.nth ) {
             int old_num = master_th->th.th_team_nproc;
             int new_num = master_th->th.th_teams_size.nth;
             kmp_info_t **other_threads = team->t.t_threads;
-            kmp_task_team_t * task_team = master_th->th.th_task_team;
             team->t.t_nproc = new_num;
-            if ( task_team ) { // task team might have lesser value of counters
-                task_team->tt.tt_ref_ct = new_num - 1;
-                task_team->tt.tt_unfinished_threads = new_num;
-            }
             for ( i = 0; i < old_num; ++i ) {
                 other_threads[i]->th.th_team_nproc = new_num;
             }
             // Adjust states of non-used threads of the team
             for ( i = old_num; i < new_num; ++i ) {
                 // Re-initialize thread's barrier data.
                 int b;
                 kmp_balign_t * balign = other_threads[i]->th.th_bar;
#endif /* OMP_40_ENABLED */
     if( parent_team->t.t_serialized &&
         parent_team != master_th->th.th_serial_team &&
         parent_team != root->r.r_root_team ) {
             __kmp_free_team( root, master_th->th.th_serial_team USE_NESTED_HOT_ARG(NULL) );
             master_th->th.th_serial_team = parent_team;
     }
 
     if ( __kmp_tasking_mode != tskm_immediate_exec ) {
-        // Restore task state from memo stack
+        if (master_th->th.th_task_state_top > 0) { // Restore task state from memo stack
             KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack);
-        if (master_th->th.th_task_state_top > 0) {
+            // Remember master's state if we re-use this nested hot team
+            master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top] = master_th->th.th_task_state;
             --master_th->th.th_task_state_top; // pop
+            // Now restore state at this level
             master_th->th.th_task_state = master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top];
         }
-        // Copy the first task team from the new child / old parent team to the thread and reset state flag.
+        // Copy the task team from the parent team to the master thread
         master_th->th.th_task_team = parent_team->t.t_task_team[master_th->th.th_task_state];
-
         KA_TRACE( 20, ( "__kmp_join_call: Master T#%d restoring task_team %p / team %p\n",
-                        __kmp_gtid_from_thread( master_th ), master_th->th.th_task_team,
-                        parent_team ) );
+                        __kmp_gtid_from_thread( master_th ), master_th->th.th_task_team, parent_team ) );
     }
 
      // TODO: GEH - cannot do this assertion because root thread not set up as executing
      // KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 0 );
      master_th->th.th_current_task->td_flags.executing = 1;
 
     __kmp_release_bootstrap_lock( &__kmp_forkjoin_lock );
 

 #endif
     ) {
         kmp_team_t *hot_team = root->r.r_hot_team;
         int f;
 
         __kmp_acquire_bootstrap_lock( &__kmp_forkjoin_lock );
 
 
-        if ( __kmp_tasking_mode != tskm_immediate_exec ) {
-            int tt_idx;
-            for (tt_idx=0; tt_idx<2; ++tt_idx) {
-                kmp_task_team_t *task_team = hot_team->t.t_task_team[tt_idx];
-                if ( ( task_team != NULL ) && TCR_SYNC_4(task_team->tt.tt_active) ) {
-                    // Signal worker threads (esp. the extra ones) to stop looking for tasks while spin waiting.
-                    // The task teams are reference counted and will be deallocated by the last worker thread.
-                    KMP_DEBUG_ASSERT( hot_team->t.t_nproc > 1 );
-                    TCW_SYNC_4( task_team->tt.tt_active, FALSE );
-                    KMP_MB();
-                    KA_TRACE( 20, ( "__kmp_set_num_threads: setting task_team %p to NULL\n",
-                                    &hot_team->t.t_task_team[tt_idx] ) );
-                    hot_team->t.t_task_team[tt_idx] = NULL;
-                }
-                else {
-                    KMP_DEBUG_ASSERT( task_team == NULL );
-                }
-            }
-        }
-
-        //
         // Release the extra threads we don't need any more.
-        //
         for ( f = new_nth;  f < hot_team->t.t_nproc; f++ ) {
             KMP_DEBUG_ASSERT( hot_team->t.t_threads[f] != NULL );
+            if ( __kmp_tasking_mode != tskm_immediate_exec) {
+                // When decreasing team size, threads no longer in the team should unref task team.
+                hot_team->t.t_threads[f]->th.th_task_team = NULL;
+            }
             __kmp_free_thread( hot_team->t.t_threads[f] );
             hot_team->t.t_threads[f] =  NULL;
         }
         hot_team->t.t_nproc = new_nth;
 #if KMP_NESTED_HOT_TEAMS
         if( thread->th.th_hot_teams ) {
             KMP_DEBUG_ASSERT( hot_team == thread->th.th_hot_teams[0].hot_team );
             thread->th.th_hot_teams[0].hot_team_nth = new_nth;
#endif
 
     /* setup the thread's cache of the team structure */
     this_thr->th.th_team_nproc      = team->t.t_nproc;
     this_thr->th.th_team_master     = master;
     this_thr->th.th_team_serialized = team->t.t_serialized;
     TCW_PTR(this_thr->th.th_sleep_loc, NULL);
 
     KMP_DEBUG_ASSERT( team->t.t_implicit_task_taskdata );
-    this_thr->th.th_task_state = 0;
 
     KF_TRACE( 10, ( "__kmp_initialize_info1: T#%d:%d this_thread=%p curtask=%p\n",
                     tid, gtid, this_thr, this_thr->th.th_current_task ) );
 
     __kmp_init_implicit_task( this_thr->th.th_team_master->th.th_ident, this_thr, team, tid, TRUE );
 
     KF_TRACE( 10, ( "__kmp_initialize_info2: T#%d:%d this_thread=%p curtask=%p\n",
                     tid, gtid, this_thr, this_thr->th.th_current_task ) );
/* Initialize dynamic dispatch */
 
         dispatch->th_deo_fcn = 0;             /* ORDERED     */
         dispatch->th_dxo_fcn = 0;             /* END ORDERED */
     }
 
     this_thr->th.th_next_pool = NULL;
 
     if (!this_thr->th.th_task_state_memo_stack) {
+        size_t i;
         this_thr->th.th_task_state_memo_stack = (kmp_uint8 *) __kmp_allocate( 4*sizeof(kmp_uint8) );
         this_thr->th.th_task_state_top = 0;
         this_thr->th.th_task_state_stack_sz = 4;
+        for (i=0; i<this_thr->th.th_task_state_stack_sz; ++i) // zero init the stack
+            this_thr->th.th_task_state_memo_stack[i] = 0;
     }
 
     KMP_DEBUG_ASSERT( !this_thr->th.th_spin_here );
     KMP_DEBUG_ASSERT( this_thr->th.th_next_waiting == 0 );
 
     KMP_MB();
 }
 
if ( __kmp_thread_pool ) {
         KMP_DEBUG_ASSERT( __kmp_thread_pool_nth >= 0 );
 
         /* setup the thread structure */
         __kmp_initialize_info( new_thr, team, new_tid, new_thr->th.th_info.ds.ds_gtid );
         KMP_DEBUG_ASSERT( new_thr->th.th_serial_team );
 
         TCW_4(__kmp_nth, __kmp_nth + 1);
 
+        new_thr->th.th_task_state = 0;
         new_thr->th.th_task_state_top = 0;
         new_thr->th.th_task_state_stack_sz = 4;
 
 #ifdef KMP_ADJUST_BLOCKTIME
         /* Adjust blocktime back to zero if necessar      y */
         /* Middle initialization might not have occurred yet */
         if ( !__kmp_env_blocktime && ( __kmp_avail_proc > 0 ) ) {
             if ( __kmp_nth > __kmp_avail_proc ) {
# else
             }
 # endif /* KMP_AFFINITY_SUPPORTED */
 #endif /* OMP_40_ENABLED */
         }
         else if( team->t.t_nproc > new_nproc ) {
             KA_TRACE( 20, ("__kmp_allocate_team: decreasing hot team thread count to %d\n", new_nproc ));
 
             team->t.t_size_changed = 1;
-            if ( __kmp_tasking_mode != tskm_immediate_exec ) {
-                // Signal the worker threads (esp. extra ones) to stop looking for tasks while spin waiting.
-                // The task teams are reference counted and will be deallocated by the last worker thread.
-                int tt_idx;
-                for (tt_idx=0; tt_idx<2; ++tt_idx) {
-                    // We don't know which of the two task teams workers are waiting on, so deactivate both.
-                    kmp_task_team_t *task_team = team->t.t_task_team[tt_idx];
-                    if ( ( task_team != NULL ) && TCR_SYNC_4(task_team->tt.tt_active) ) {
-                        KMP_DEBUG_ASSERT( team->t.t_nproc > 1 );
-                        TCW_SYNC_4( task_team->tt.tt_active, FALSE );
-                        KMP_MB();
-                        KA_TRACE(20, ("__kmp_allocate_team: setting task_team %p to NULL\n",
-                                      &team->t.t_task_team[tt_idx]));
-                        team->t.t_task_team[tt_idx] = NULL;
-                    }
-                    else {
-                        KMP_DEBUG_ASSERT( task_team == NULL );
-                    }
-                }
-            }
 #if KMP_NESTED_HOT_TEAMS
             if( __kmp_hot_teams_mode == 0 ) {
                 // AC: saved number of threads should correspond to team's value in this mode,
                 // can be bigger in mode 1, when hot team has some threads in reserve
                 KMP_DEBUG_ASSERT(hot_teams[level].hot_team_nth == team->t.t_nproc);
                 hot_teams[level].hot_team_nth = new_nproc;
 #endif // KMP_NESTED_HOT_TEAMS
                 /* release the extra threads we don't need any more */
                 for( f = new_nproc  ;  f < team->t.t_nproc  ;  f++ ) {
                     KMP_DEBUG_ASSERT( team->t.t_threads[ f ] );
+                    if ( __kmp_tasking_mode != tskm_immediate_exec) {
+                        // When decreasing team size, threads no longer in the team should unref task team.
+                        team->t.t_threads[f]->th.th_task_team = NULL;
+                    }
                     __kmp_free_thread( team->t.t_threads[ f ] );
                     team->t.t_threads[ f ] = NULL;
                 }
 #if KMP_NESTED_HOT_TEAMS
             } // (__kmp_hot_teams_mode == 0)
 #endif // KMP_NESTED_HOT_TEAMS
             team->t.t_nproc =  new_nproc;
             // TODO???: team->t.t_max_active_levels = new_max_active_levels;
             team->t.t_sched =  new_icvs->sched;
             __kmp_reinitialize_team( team, new_icvs, root->r.r_uber_thread->th.th_ident );
 
-            if ( __kmp_tasking_mode != tskm_immediate_exec ) {
-                // Init both task teams
-                int tt_idx;
-                for (tt_idx=0; tt_idx<2; ++tt_idx) {
-                    kmp_task_team_t *task_team = team->t.t_task_team[tt_idx];
-                    if ( task_team != NULL ) {
-                        KMP_DEBUG_ASSERT( ! TCR_4(task_team->tt.tt_found_tasks) );
-                        task_team->tt.tt_nproc = new_nproc;
-                        task_team->tt.tt_unfinished_threads = new_nproc;
-                        task_team->tt.tt_ref_ct = new_nproc - 1;
-                    }
-                }
-            }
-
             /* update the remaining threads */
-            if (level) {
-                team->t.t_threads[0]->th.th_team_nproc = new_nproc;
-                for(f = 1; f < new_nproc; ++f) {
-                    team->t.t_threads[f]->th.th_team_nproc = new_nproc;
-                    team->t.t_threads[f]->th.th_task_state = 0;
-                }
-            }
-            else {
-                for(f = 0; f < new_nproc; ++f) {
-                    team->t.t_threads[f]->th.th_team_nproc = new_nproc;
-                }
-            }
             // restore the current task state of the master thread: should be the implicit task
             KF_TRACE( 10, ("__kmp_allocate_team: T#%d, this_thread=%p team=%p\n",
                        0, team->t.t_threads[0], team ) );
 
             __kmp_push_current_task_to_thread( team->t.t_threads[ 0 ], team, 0 );
 
 #ifdef KMP_DEBUG
             for ( f = 0; f < team->t.t_nproc; f++ ) {
#if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED
                 __kmp_set_system_affinity( old_mask, TRUE );
                 KMP_CPU_FREE(old_mask);
             }
 #endif
 #if KMP_NESTED_HOT_TEAMS
             } // end of check of t_nproc vs. new_nproc vs. hot_team_nth
 #endif // KMP_NESTED_HOT_TEAMS
             /* make sure everyone is syncronized */
+            int old_nproc = team->t.t_nproc; // save old value and use to update only new threads below
             __kmp_initialize_team( team, new_nproc, new_icvs, root->r.r_uber_thread->th.th_ident );
 
-            if ( __kmp_tasking_mode != tskm_immediate_exec ) {
-                // Signal the worker threads to stop looking for tasks while spin waiting.
-                // The task teams are reference counted and will be deallocated by the last worker thread.
-                int tt_idx;
-                for (tt_idx=0; tt_idx<2; ++tt_idx) {
-                    // We don't know which of the two task teams workers are waiting on, so deactivate both.
-                    kmp_task_team_t *task_team = team->t.t_task_team[tt_idx];
-                    if ( (task_team != NULL) && TCR_SYNC_4(task_team->tt.tt_active) ) {
-                        TCW_SYNC_4( task_team->tt.tt_active, FALSE );
-                        team->t.t_task_team[tt_idx] = NULL;
-                    }
-                }
-            }
-
             /* reinitialize the threads */
             KMP_DEBUG_ASSERT(team->t.t_nproc == new_nproc);
-            if (level) {
-                int old_state = team->t.t_threads[0]->th.th_task_state;
-                for (f=0;  f < team->t.t_nproc; ++f)
-                    __kmp_initialize_info( team->t.t_threads[ f ], team, f, __kmp_gtid_from_tid( f, team ) );
-                // th_task_state for master thread will be put in stack of states in __kmp_fork_call()
-                // before zeroing, for workers it was just zeroed in __kmp_initialize_info()
-                team->t.t_threads[0]->th.th_task_state = old_state;
-            }
-            else {
-                int old_state = team->t.t_threads[0]->th.th_task_state;
-                for (f=0;  f<team->t.t_nproc; ++f) {
-                    __kmp_initialize_info( team->t.t_threads[ f ], team, f, __kmp_gtid_from_tid( f, team ) );
+            if (level) { // set th_task_state for new threads in nested hot team
+                // __kmp_initialize_info() no longer zeroes th_task_state, so we should only need to set the
+                // th_task_state for the new threads. th_task_state for master thread will not be accurate until 
+                // after this in __kmp_fork_call(), so we look to the master's memo_stack to get the correct value.
+                for (f=old_nproc; f < team->t.t_nproc; ++f)
+                    team->t.t_threads[f]->th.th_task_state = team->t.t_threads[0]->th.th_task_state_memo_stack[level];
+            }
+            else { // set th_task_state for new threads in non-nested hot team
+                int old_state = team->t.t_threads[0]->th.th_task_state; // copy master's state
+                for (f=old_nproc; f < team->t.t_nproc; ++f)
                     team->t.t_threads[f]->th.th_task_state = old_state;
-                    team->t.t_threads[f]->th.th_task_team = team->t.t_task_team[old_state];
-                }
             }
 
 #ifdef KMP_DEBUG
             for ( f = 0; f < team->t.t_nproc; ++ f ) {
                 KMP_DEBUG_ASSERT( team->t.t_threads[f] &&
                     team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc );
             }
 #endif
#endif // KMP_NESTED_HOT_TEAMS
     /* team is done working */
     TCW_SYNC_PTR(team->t.t_pkfn, NULL); // Important for Debugging Support Library.
     team->t.t_copyin_counter = 0; // init counter for possible reuse
     // Do not reset pointer to parent team to NULL for hot teams.
 
     /* if we are non-hot team, release our threads */
     if( ! use_hot_team ) {
         if ( __kmp_tasking_mode != tskm_immediate_exec ) {
+            // Delete task teams
             int tt_idx;
             for (tt_idx=0; tt_idx<2; ++tt_idx) {
-                // We don't know which of the two task teams workers are waiting on, so deactivate both.
                 kmp_task_team_t *task_team = team->t.t_task_team[tt_idx];
                 if ( task_team != NULL ) {
-                    // Signal the worker threads to stop looking for tasks while spin waiting.  The task
-                    // teams are reference counted and will be deallocated by the last worker thread via the
-                    // thread's pointer to the task team.
-                    KA_TRACE( 20, ( "__kmp_free_team: deactivating task_team %p\n", task_team ) );
+                    for (f=0; f<team->t.t_nproc; ++f) { // Have all threads unref task teams
+                        team->t.t_threads[f]->th.th_task_team = NULL;
+                    }
+                    KA_TRACE( 20, ( "__kmp_free_team: T#%d deactivating task_team %p on team %d\n", __kmp_get_gtid(), task_team, team->t.t_id ) );
                     KMP_DEBUG_ASSERT( team->t.t_nproc > 1 );
-                    TCW_SYNC_4( task_team->tt.tt_active, FALSE );
-                    KMP_MB();
+                    __kmp_free_task_team( master, task_team );
                     team->t.t_task_team[tt_idx] = NULL;
                 }
             }
         }
 
         // Reset pointer to parent team only for non-hot teams.
         team->t.t_parent = NULL;
 
__kmp_free_thread( kmp_info_t *this_th )
     // When moving thread to pool, switch thread to wait on own b_go flag, and uninitialized (NULL team).
     int b;
     kmp_balign_t *balign = this_th->th.th_bar;
     for (b=0; b<bs_last_barrier; ++b) {
         if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG)
             balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG;
         balign[b].bb.team = NULL;
     }
+    this_th->th.th_task_state = 0;
 
 
     /* put thread back on the free pool */
     TCW_PTR(this_th->th.th_team, NULL);
     TCW_PTR(this_th->th.th_root, NULL);
     TCW_PTR(this_th->th.th_dispatch, NULL);               /* NOT NEEDED */
 
     //

 
 #if OMPT_SUPPORT
     if (ompt_enabled &&
         ompt_callbacks.ompt_callback(ompt_event_thread_end)) {
         __ompt_thread_end(ompt_thread_worker, gtid);
     }
 #endif
 
-    if ( TCR_PTR( this_thr->th.th_task_team ) != NULL ) {
-        __kmp_unref_task_team( this_thr->th.th_task_team, this_thr );
-    }
+    this_thr->th.th_task_team = NULL;
     /* run the destructors for the threadprivate data for this thread */
     __kmp_common_destroy_gtid( gtid );
 
     KA_TRACE( 10, ("__kmp_launch_thread: T#%d done\n", gtid ) );
     KMP_MB();
     return this_thr;
 }
 
/* find out who we are and what we should do */
                 KA_TRACE( 10, ("__kmp_internal_end_thread: unregistering sibling T#%d\n", gtid ));
                 __kmp_unregister_root_current_thread( gtid );
             }
         } else {
             /* just a worker thread, let's leave */
             KA_TRACE( 10, ("__kmp_internal_end_thread: worker thread T#%d\n", gtid ));
 
             if ( gtid >= 0 ) {
-                kmp_info_t *this_thr = __kmp_threads[ gtid ];
-                if (TCR_PTR(this_thr->th.th_task_team) != NULL) {
-                    __kmp_unref_task_team(this_thr->th.th_task_team, this_thr);
-                }
+                __kmp_threads[gtid]->th.th_task_team = NULL;
             }
 
             KA_TRACE( 10, ("__kmp_internal_end_thread: worker thread done, exiting T#%d\n", gtid ));
             return;
         }
     }
     #if defined KMP_DYNAMIC_LIB
     // AC: lets not shutdown the Linux* OS dynamic library at the exit of uber thread,

openmp/trunk/runtime/src/kmp_tasking.c

if ( flags->proxy == TASK_PROXY ) {
 
         /* are we running in a sequential parallel or tskm_immediate_exec... we need tasking support enabled */
         if ( (thread->th.th_task_team) == NULL ) {
             /* This should only happen if the team is serialized
                 setup a task team and propagate it to the thread
             */
             KMP_DEBUG_ASSERT(team->t.t_serialized);
             KA_TRACE(30,("T#%d creating task team in __kmp_task_alloc for proxy task\n", gtid));
-            __kmp_task_team_setup(thread,team,0,1); // 0,1 indicates only setup the current team regardless of nthreads
+            __kmp_task_team_setup(thread,team,1); // 1 indicates setup the current team regardless of nthreads
             thread->th.th_task_team = team->t.t_task_team[thread->th.th_task_state];
         }
         kmp_task_team_t * task_team = thread->th.th_task_team;
 
         /* tasking must be enabled now as the task might not be pushed */
         if ( !KMP_TASKING_ENABLED( task_team ) ) {
             KA_TRACE(30,("T#%d enabling tasking in __kmp_task_alloc for proxy task\n", gtid));
             __kmp_enable_tasking( task_team, thread );

 
 kmp_int32
 __kmpc_omp_taskwait( ident_t *loc_ref, kmp_int32 gtid )
 {
     kmp_taskdata_t * taskdata;
     kmp_info_t * thread;
     int thread_finished = FALSE;
 
-    KA_TRACE(10, ("__kmpc_omp_taskwait(enter): T#%d loc=%p\n",
-                  gtid, loc_ref) );
+    KA_TRACE(10, ("__kmpc_omp_taskwait(enter): T#%d loc=%p\n", gtid, loc_ref) );
 
     if ( __kmp_tasking_mode != tskm_immediate_exec ) {
         // GEH TODO: shouldn't we have some sort of OMPRAP API calls here to mark begin wait?
 
         thread = __kmp_threads[ gtid ];
         taskdata = thread -> th.th_current_task;
 #if USE_ITT_BUILD
         // Note: These values are used by ITT events as well.
static inline int __kmp_execute_tasks_template(kmp_info_t *thread, kmp_int32 gtid, C *flag, int final_spin,
     kmp_taskdata_t *      current_task = thread -> th.th_current_task;
     volatile kmp_uint32 * unfinished_threads;
     kmp_int32             nthreads, last_stolen, k, tid;
 
     KMP_DEBUG_ASSERT( __kmp_tasking_mode != tskm_immediate_exec );
     KMP_DEBUG_ASSERT( thread == __kmp_threads[ gtid ] );
 
     task_team = thread -> th.th_task_team;
-    KMP_DEBUG_ASSERT( task_team != NULL );
+    if (task_team == NULL) return FALSE;
 
     KA_TRACE(15, ("__kmp_execute_tasks_template(enter): T#%d final_spin=%d *thread_finished=%d\n",
                   gtid, final_spin, *thread_finished) );
 
     threads_data = (kmp_thread_data_t *)TCR_PTR(task_team -> tt.tt_threads_data);
     KMP_DEBUG_ASSERT( threads_data != NULL );
 
     nthreads = task_team -> tt.tt_nproc;
#endif /* USE_ITT_BUILD */
         // is met, then return now, so that the barrier gather/release pattern can proceed.
         // If this thread is in the last spin loop in the barrier, waiting to be
         // released, we know that the termination condition will not be satisified,
         // so don't waste any cycles checking it.
         if (flag == NULL || (!final_spin && flag->done_check())) {
             KA_TRACE(15, ("__kmp_execute_tasks_template(exit #1): T#%d spin condition satisfied\n", gtid) );
             return TRUE;
         }
+        if (thread->th.th_task_team == NULL) break;
         KMP_YIELD( __kmp_library == library_throughput );   // Yield before executing next task
     }
 
     // This thread's work queue is empty.  If we are in the final spin loop
     // of the barrier, check and see if the termination condition is satisfied.
 #if OMP_41_ENABLED
     // The work queue may be empty but there might be proxy tasks still executing
     if (final_spin && TCR_4(current_task -> td_incomplete_child_tasks) == 0) 
#endif
         // th.th_team field for the next parallel region.
         // If we can steal more work, we know that this has not happened yet.
         if (flag != NULL && flag->done_check()) {
             KA_TRACE(15, ("__kmp_execute_tasks_template(exit #2): T#%d spin condition satisfied\n", gtid) );
             return TRUE;
         }
     }
 
+    if (thread->th.th_task_team == NULL) return FALSE;
 #if OMP_41_ENABLED
     // check if there are other threads to steal from, otherwise go back
     if ( nthreads  == 1 )
         goto start;
 #endif
 
     // Try to steal from the last place I stole from successfully.
     tid = thread -> th.th_info.ds.ds_tid;//__kmp_tid_from_gtid( gtid );
#endif /* USE_ITT_BUILD */
 
             // Check to see if this thread can proceed.
             if (flag == NULL || (!final_spin && flag->done_check())) {
                 KA_TRACE(15, ("__kmp_execute_tasks_template(exit #3): T#%d spin condition satisfied\n",
                               gtid) );
                 return TRUE;
             }
 
+            if (thread->th.th_task_team == NULL) break;
             KMP_YIELD( __kmp_library == library_throughput );   // Yield before executing next task
             // If the execution of the stolen task resulted in more tasks being
             // placed on our run queue, then restart the whole process.
             if (TCR_4(threads_data[ tid ].td.td_deque_ntasks) != 0) {
                 KA_TRACE(20, ("__kmp_execute_tasks_template: T#%d stolen task spawned other tasks, restart\n",
                               gtid) );
                 goto start;
             }
#endif
             // th.th_team field for the next parallel region.
             // If we can steal more work, we know that this has not happened yet.
             if (flag != NULL && flag->done_check()) {
                 KA_TRACE(15, ("__kmp_execute_tasks_template(exit #4): T#%d spin condition satisfied\n",
                               gtid) );
                 return TRUE;
             }
         }
+        if (thread->th.th_task_team == NULL) return FALSE;
     }
 
     // Find a different thread to steal work from.  Pick a random thread.
     // My initial plan was to cycle through all the threads, and only return
     // if we tried to steal from every thread, and failed.  Arch says that's
     // not such a great idea.
     // GEH - need yield code in this loop for throughput library mode?
     new_victim:
#endif /* USE_ITT_BUILD */
             }
 
             // Check to see if this thread can proceed.
             if (flag == NULL || (!final_spin && flag->done_check())) {
                 KA_TRACE(15, ("__kmp_execute_tasks_template(exit #5): T#%d spin condition satisfied\n",
                               gtid) );
                 return TRUE;
             }
+            if (thread->th.th_task_team == NULL) break;
             KMP_YIELD( __kmp_library == library_throughput );   // Yield before executing next task
 
             // If the execution of the stolen task resulted in more tasks being
             // placed on our run queue, then restart the whole process.
             if (TCR_4(threads_data[ tid ].td.td_deque_ntasks) != 0) {
                 KA_TRACE(20, ("__kmp_execute_tasks_template: T#%d stolen task spawned other tasks, restart\n",
                               gtid) );
                 goto start;
#endif
             // thread to pass through the barrier, where it might reset each thread's
             // th.th_team field for the next parallel region.
             // If we can steal more work, we know that this has not happened yet.
             if (flag != NULL && flag->done_check()) {
                 KA_TRACE(15, ("__kmp_execute_tasks_template(exit #6): T#%d spin condition satisfied\n", gtid) );
                 return TRUE;
             }
         }
+        if (thread->th.th_task_team == NULL) return FALSE;
     }
 
     KA_TRACE(15, ("__kmp_execute_tasks_template(exit #7): T#%d can't find work\n", gtid) );
     return FALSE;
 }
 
 int __kmp_execute_tasks_32(kmp_info_t *thread, kmp_int32 gtid, kmp_flag_32 *flag, int final_spin,
                            int *thread_finished
__kmp_allocate_task_team( kmp_info_t *thread, kmp_team_t *team )
     TCW_4(task_team -> tt.tt_found_tasks, FALSE);
 #if OMP_41_ENABLED
     TCW_4(task_team -> tt.tt_found_proxy_tasks, FALSE);
 #endif
     task_team -> tt.tt_nproc = nthreads = team->t.t_nproc;
 
     TCW_4( task_team -> tt.tt_unfinished_threads, nthreads );
     TCW_4( task_team -> tt.tt_active, TRUE );
-    TCW_4( task_team -> tt.tt_ref_ct, nthreads - 1);
 
-    KA_TRACE( 20, ( "__kmp_allocate_task_team: T#%d exiting; task_team = %p\n",
-                    (thread ? __kmp_gtid_from_thread( thread ) : -1), task_team ) );
+    KA_TRACE( 20, ( "__kmp_allocate_task_team: T#%d exiting; task_team = %p unfinished_threads init'd to %d\n",
+                    (thread ? __kmp_gtid_from_thread( thread ) : -1), task_team, task_team -> tt.tt_unfinished_threads) );
     return task_team;
 }
 
 
 //------------------------------------------------------------------------------
 // __kmp_free_task_team:
 // Frees the task team associated with a specific thread, and adds it
 // to the global task team free list.
-//
 
-static void
+void
 __kmp_free_task_team( kmp_info_t *thread, kmp_task_team_t *task_team )
 {
     KA_TRACE( 20, ( "__kmp_free_task_team: T#%d task_team = %p\n",
                     thread ? __kmp_gtid_from_thread( thread ) : -1, task_team ) );
 
-    KMP_DEBUG_ASSERT( TCR_4(task_team -> tt.tt_ref_ct) == 0 );
-
     // Put task team back on free list
     __kmp_acquire_bootstrap_lock( & __kmp_task_team_lock );
 
     KMP_DEBUG_ASSERT( task_team -> tt.tt_next == NULL );
     task_team -> tt.tt_next = __kmp_free_task_teams;
     TCW_4(task_team -> tt.tt_found_tasks, FALSE);
     TCW_PTR(__kmp_free_task_teams, task_team);
 
if ( TCR_PTR(__kmp_free_task_teams) != NULL ) {
                 __kmp_free_task_threads_data( task_team );
             }
             __kmp_free( task_team );
         }
         __kmp_release_bootstrap_lock( &__kmp_task_team_lock );
     }
 }
 
-
-//------------------------------------------------------------------------------
-// __kmp_unref_task_teams:
-// Remove one thread from referencing the task team structure by
-// decreasing the reference count and deallocate task team if no more
-// references to it.
-//
-void
-__kmp_unref_task_team( kmp_task_team_t *task_team, kmp_info_t *thread )
-{
-    kmp_uint ref_ct;
-
-    ref_ct = KMP_TEST_THEN_DEC32( (kmp_int32 *)(& task_team->tt.tt_ref_ct) ) - 1;
-
-    KA_TRACE( 20, ( "__kmp_unref_task_team: T#%d task_team = %p ref_ct = %d\n",
-                    __kmp_gtid_from_thread( thread ), task_team, ref_ct ) );
-
-
-    if ( ref_ct == 0 ) {
-        __kmp_free_task_team( thread, task_team );
-    }
-
-    TCW_PTR( *((volatile kmp_task_team_t **)(&thread->th.th_task_team)), NULL );
-}
-
-
 //------------------------------------------------------------------------------
 // __kmp_wait_to_unref_task_teams:
 // Some threads could still be in the fork barrier release code, possibly
 // trying to steal tasks.  Wait for each thread to unreference its task team.
 //
 void
 __kmp_wait_to_unref_task_teams(void)
 {
#endif
             if ( TCR_PTR(thread->th.th_task_team) == NULL ) {
                 KA_TRACE( 10, ("__kmp_wait_to_unref_task_team: T#%d task_team == NULL\n",
                                __kmp_gtid_from_thread( thread ) ) );
                 continue;
             }
 #if KMP_OS_WINDOWS
             // TODO: GEH - add this check for Linux* OS / OS X* as well?
             if (!__kmp_is_thread_alive(thread, &exit_val)) {
-                if (TCR_PTR(thread->th.th_task_team) != NULL) {
-                    __kmp_unref_task_team( thread->th.th_task_team, thread );
-                }
+                thread->th.th_task_team = NULL;
                 continue;
             }
 #endif
 
             done = FALSE;  // Because th_task_team pointer is not NULL for this thread
 
             KA_TRACE( 10, ("__kmp_wait_to_unref_task_team: Waiting for T#%d to unreference task_team\n",
                            __kmp_gtid_from_thread( thread ) ) );

 }
 
 
 //------------------------------------------------------------------------------
 // __kmp_task_team_setup:  Create a task_team for the current team, but use
 // an already created, unused one if it already exists.
 // This may be called by any thread, but only for teams with # threads >1.
 void
-__kmp_task_team_setup( kmp_info_t *this_thr, kmp_team_t *team, int both, int always )
+__kmp_task_team_setup( kmp_info_t *this_thr, kmp_team_t *team, int always )
 {
     KMP_DEBUG_ASSERT( __kmp_tasking_mode != tskm_immediate_exec );
 
-    if ( ( team->t.t_task_team[this_thr->th.th_task_state] == NULL ) && ( always || team->t.t_nproc > 1 ) ) {
-        // Allocate a new task team, which will be propagated to
-        // all of the worker threads after the barrier.  As they
-        // spin in the barrier release phase, then will continue
-        // to use the previous task team struct, until they receive
-        // the signal to stop checking for tasks (they can't safely
-        // reference the kmp_team_t struct, which could be reallocated
-        // by the master thread).
+    // If this task_team hasn't been created yet, allocate it. It will be used in the region after the next.
+    // If it exists, it is the current task team and shouldn't be touched yet as it may still be in use.
+    if (team->t.t_task_team[this_thr->th.th_task_state] == NULL && (always || team->t.t_nproc > 1) ) { 
         team->t.t_task_team[this_thr->th.th_task_state] = __kmp_allocate_task_team( this_thr, team );
         KA_TRACE(20, ("__kmp_task_team_setup: Master T#%d created new task_team %p for team %d at parity=%d\n",
                       __kmp_gtid_from_thread(this_thr), team->t.t_task_team[this_thr->th.th_task_state],
                       ((team != NULL) ? team->t.t_id : -1), this_thr->th.th_task_state));
     }
-    // else: Either all threads have reported in, and no tasks were spawned for this release->gather region
-    // Leave the old task team struct in place for the upcoming region.
-    // No task teams are formed for serialized teams.
-    if (both) {
-        int other_team = 1 - this_thr->th.th_task_state;
-        if ( ( team->t.t_task_team[other_team] == NULL ) && ( team->t.t_nproc > 1 ) ) { // setup other team as well
-            team->t.t_task_team[other_team] = __kmp_allocate_task_team( this_thr, team );
-            KA_TRACE(20, ("__kmp_task_team_setup: Master T#%d created second new task_team %p for team %d at parity=%d\n",
-                          __kmp_gtid_from_thread( this_thr ), team->t.t_task_team[other_team],
-                          ((team != NULL) ? team->t.t_id : -1), other_team ));
-        }
+
+    // After threads exit the release, they will call sync, and then point to this other task_team; make sure it is 
+    // allocated and properly initialized. As threads spin in the barrier release phase, they will continue to use the
+    // previous task_team struct(above), until they receive the signal to stop checking for tasks (they can't safely
+    // reference the kmp_team_t struct, which could be reallocated by the master thread). No task teams are formed for 
+    // serialized teams.
+    if (team->t.t_task_team[other_team] == NULL && team->t.t_nproc > 1) { // setup other team as well
+    else { // Leave the old task team struct in place for the upcoming region; adjust as needed
+        kmp_task_team_t *task_team = team->t.t_task_team[other_team];
+        if (!task_team->tt.tt_active || team->t.t_nproc != task_team->tt.tt_nproc) {
+            TCW_4(task_team->tt.tt_nproc, team->t.t_nproc);
+            TCW_4(task_team->tt.tt_found_tasks, FALSE);
+#if OMP_41_ENABLED
+            TCW_4(task_team->tt.tt_found_proxy_tasks, FALSE);
+#endif
+            TCW_4(task_team->tt.tt_unfinished_threads, team->t.t_nproc );
+            TCW_4(task_team->tt.tt_active, TRUE );
+        }
+        // if team size has changed, the first thread to enable tasking will realloc threads_data if necessary
+        KA_TRACE(20, ("__kmp_task_team_setup: Master T#%d reset next task_team %p for team %d at parity=%d\n",
+                      __kmp_gtid_from_thread( this_thr ), team->t.t_task_team[other_team],
+                      ((team != NULL) ? team->t.t_id : -1), other_team ));
     }
 }
 
 
 //------------------------------------------------------------------------------
 // __kmp_task_team_sync: Propagation of task team data from team to threads
 // which happens just after the release phase of a team barrier.  This may be
 // called by any thread, but only for teams with # threads > 1.
 
 void
 __kmp_task_team_sync( kmp_info_t *this_thr, kmp_team_t *team )
 {
     KMP_DEBUG_ASSERT( __kmp_tasking_mode != tskm_immediate_exec );
 
-    // In case this thread never saw that the task team was no longer active, unref/deallocate it now.
-    if ( this_thr->th.th_task_team != NULL ) {
-        if ( ! TCR_SYNC_4( this_thr->th.th_task_team->tt.tt_active ) ) {
-            KMP_DEBUG_ASSERT( ! KMP_MASTER_TID( __kmp_tid_from_gtid( __kmp_gtid_from_thread( this_thr ) ) ) );
-            KA_TRACE(20, ("__kmp_task_team_sync: Thread T#%d task team (%p)is not active, unrefing\n",
-                          __kmp_gtid_from_thread( this_thr ), this_thr->th.th_task_team));
-            __kmp_unref_task_team( this_thr->th.th_task_team, this_thr );
-        } 
-#if KMP_DEBUG       
-        else {  // We are re-using a task team that was never enabled.
-            KMP_DEBUG_ASSERT(this_thr->th.th_task_team == team->t.t_task_team[this_thr->th.th_task_state]);
-        }
-#endif
-    }
-
     // Toggle the th_task_state field, to switch which task_team this thread refers to
     this_thr->th.th_task_state = 1 - this_thr->th.th_task_state;
     // It is now safe to propagate the task team pointer from the team struct to the current thread.
     TCW_PTR(this_thr->th.th_task_team, team->t.t_task_team[this_thr->th.th_task_state]);
-    KA_TRACE(20, ("__kmp_task_team_sync: Thread T#%d task team switched to %p from Team #%d task team (parity=%d)\n",
+    KA_TRACE(20, ("__kmp_task_team_sync: Thread T#%d task team switched to task_team %p from Team #%d (parity=%d)\n",
                   __kmp_gtid_from_thread( this_thr ), this_thr->th.th_task_team,
                   ((team != NULL) ? team->t.t_id : -1), this_thr->th.th_task_state));
 }
 
 
 //--------------------------------------------------------------------------------------------
 // __kmp_task_team_wait: Master thread waits for outstanding tasks after the barrier gather
-// phase.  Only called by master thread if #threads in team > 1 or if proxy tasks were created
+// phase.  Only called by master thread if #threads in team > 1 or if proxy tasks were created.
+// wait is a flag that defaults to 1 (see kmp.h), but waiting can be turned off by passing in 0
+// optionally as the last argument. When wait is zero, master thread does not wait for
+// unfinished_threads to reach 0.
 void
 __kmp_task_team_wait( kmp_info_t *this_thr, kmp_team_t *team
                       USE_ITT_BUILD_ARG(void * itt_sync_obj)
-                      )
+                      , int wait)
 {
     kmp_task_team_t *task_team = team->t.t_task_team[this_thr->th.th_task_state];
 
     KMP_DEBUG_ASSERT( __kmp_tasking_mode != tskm_immediate_exec );
     KMP_DEBUG_ASSERT( task_team == this_thr->th.th_task_team );
 
     if ( ( task_team != NULL ) && KMP_TASKING_ENABLED(task_team) ) {
+        if (wait) {
             KA_TRACE(20, ("__kmp_task_team_wait: Master T#%d waiting for all tasks (for unfinished_threads to reach 0) on task_team = %p\n",
                           __kmp_gtid_from_thread(this_thr), task_team));
             // Worker threads may have dropped through to release phase, but could still be executing tasks. Wait
             // here for tasks to complete. To avoid memory contention, only master thread checks termination condition.
             kmp_flag_32 flag(&task_team->tt.tt_unfinished_threads, 0U);
             flag.wait(this_thr, TRUE
                       USE_ITT_BUILD_ARG(itt_sync_obj));
-
-        // Kill the old task team, so that the worker threads will stop referencing it while spinning.
-        // They will deallocate it when the reference count reaches zero.
-        // The master thread is not included in the ref count.
-        KA_TRACE(20, ("__kmp_task_team_wait: Master T#%d deactivating task_team %p: setting active to false, setting local and team's pointer to NULL\n",
+        }
+        // Deactivate the old task team, so that the worker threads will stop referencing it while spinning.
+        KA_TRACE(20, ("__kmp_task_team_wait: Master T#%d deactivating task_team %p: "
+                      "setting active to false, setting local and team's pointer to NULL\n",
                       __kmp_gtid_from_thread(this_thr), task_team));
 #if OMP_41_ENABLED
         KMP_DEBUG_ASSERT( task_team->tt.tt_nproc > 1 || task_team->tt.tt_found_proxy_tasks == TRUE );
         TCW_SYNC_4( task_team->tt.tt_found_proxy_tasks, FALSE );
 #else
         KMP_DEBUG_ASSERT( task_team->tt.tt_nproc > 1 );
 #endif
         TCW_SYNC_4( task_team->tt.tt_active, FALSE );
         KMP_MB();
 
         TCW_PTR(this_thr->th.th_task_team, NULL);
-        team->t.t_task_team[this_thr->th.th_task_state] = NULL;
     }
 }
 
 
 //------------------------------------------------------------------------------
 // __kmp_tasking_barrier:
 // This routine may only called when __kmp_tasking_mode == tskm_extra_barrier.
 // Internal function to execute all tasks prior to a regular barrier or a

openmp/trunk/runtime/src/kmp_wait_release.h

while (flag->notdone_check()) {
            2) All tasks have been executed to completion, this thread has decremented the task
               team's ref ct and possibly deallocated it, and should no longer reference it.
            3) Tasking is off for this region.  This could be because we are in a serialized region
               (perhaps the outer one), or else tasking was manually disabled (KMP_TASKING=0).  */
         kmp_task_team_t * task_team = NULL;
         if (__kmp_tasking_mode != tskm_immediate_exec) {
             task_team = this_thr->th.th_task_team;
             if (task_team != NULL) {
-                if (!TCR_SYNC_4(task_team->tt.tt_active)) {
-                    KMP_DEBUG_ASSERT(!KMP_MASTER_TID(this_thr->th.th_info.ds.ds_tid));
-                    __kmp_unref_task_team(task_team, this_thr);
-                } else if (KMP_TASKING_ENABLED(task_team)) {
+                if (TCR_SYNC_4(task_team->tt.tt_active)) {
+                    if (KMP_TASKING_ENABLED(task_team))
                         flag->execute_tasks(this_thr, th_gtid, final_spin, &tasks_completed
                                             USE_ITT_BUILD_ARG(itt_sync_obj), 0);
                 }
+                else {
+                    KMP_DEBUG_ASSERT(!KMP_MASTER_TID(this_thr->th.th_info.ds.ds_tid));
+                    this_thr->th.th_task_team = NULL;
+                }
             } // if
         } // if
 
         KMP_FSYNC_SPIN_PREPARE(spin);
         if (TCR_4(__kmp_global.g.g_done)) {
             if (__kmp_global.g.g_abort)
                 __kmp_abort_thread();
             break;