Bug fix for hang when tasks used in nested parallel.

[tlwilmar]

Summary

Bug fix for hang when omp task and nested parallelism used together.
Still some problem remains with task state saving/restoring, but
user's case works fine now. All tasking unit tests passed as well.

Patch by Andrey Churbanov

[jlpeyton]

All the tasking tests pass. LGTM.

runtime/src/kmp_runtime.c

if (ompt_enabled) {
         ompt_parallel_id = __ompt_parallel_id_new(gtid);
         ompt_task_id = __ompt_get_task_id_internal(0);
         ompt_frame = __ompt_get_task_frame_internal(0);
     }
 #endif
 
     // Nested level will be an index in the nested nthreads array
     level         = parent_team->t.t_level;
-#if OMP_40_ENABLED
     active_level  = parent_team->t.t_active_level; // is used to launch non-serial teams even if nested is not allowed
+#if OMP_40_ENABLED
     teams_level    = master_th->th.th_teams_level; // needed to check nesting inside the teams
 #endif
 #if KMP_NESTED_HOT_TEAMS
     p_hot_teams   = &master_th->th.th_hot_teams;
     if( *p_hot_teams == NULL && __kmp_hot_teams_max_level > 0 ) {
         *p_hot_teams = (kmp_hot_team_ptr_t*)__kmp_allocate(
                 sizeof(kmp_hot_team_ptr_t) * __kmp_hot_teams_max_level);
         (*p_hot_teams)[0].hot_team = root->r.r_hot_team;
#if 0
         // Patch out an assertion that trips while the runtime seems to operate correctly.
         // Avoiding the preconditions that cause the assertion to trip has been promised as a forthcoming patch.
         KMP_DEBUG_ASSERT(master_th->th.th_task_team == parent_team->t.t_task_team[master_th->th.th_task_state]);
 #endif
         KA_TRACE( 20, ( "__kmp_fork_call: Master T#%d pushing task_team %p / team %p, new task_team %p / team %p\n",
                       __kmp_gtid_from_thread( master_th ), master_th->th.th_task_team,
                       parent_team, team->t.t_task_team[master_th->th.th_task_state], team ) );
 
-        if ( level || master_th->th.th_task_team ) {
+        if ( active_level || master_th->th.th_task_team ) {
             // Take a memo of master's task_state
             KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack);
             if (master_th->th.th_task_state_top >= master_th->th.th_task_state_stack_sz) { // increase size
                 kmp_uint32 new_size = 2*master_th->th.th_task_state_stack_sz;
                 kmp_uint8 *old_stack, *new_stack;
                 kmp_uint32 i;
                 new_stack = (kmp_uint8 *)__kmp_allocate(new_size);
                 for (i=0; i<master_th->th.th_task_state_stack_sz; ++i) {
                     new_stack[i] = master_th->th.th_task_state_memo_stack[i];
                 }
                 for (i=master_th->th.th_task_state_stack_sz; i<new_size; ++i) { // zero-init rest of stack
                     new_stack[i] = 0;
                 }
                 old_stack = master_th->th.th_task_state_memo_stack;
                 master_th->th.th_task_state_memo_stack = new_stack;
                 master_th->th.th_task_state_stack_sz = new_size;
                 __kmp_free(old_stack);
             }
             // Store master's task_state on stack
             master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top] = master_th->th.th_task_state;
             master_th->th.th_task_state_top++;
 #if KMP_NESTED_HOT_TEAMS
-            if (team == master_th->th.th_hot_teams[level].hot_team) { // Restore master's nested state if nested hot team
+            if (team == master_th->th.th_hot_teams[active_level].hot_team) { // Restore master's nested state if nested hot team
                 master_th->th.th_task_state = master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top];
             }
             else {
 #endif
                 master_th->th.th_task_state = 0;
 #if KMP_NESTED_HOT_TEAMS
             }
 #endif