Minor fixes: moved some code under debug, format improvements

[tlwilmar]

Summary

Some code was restructured to move it under KMP_DEBUG. The rest is formatting changes to unbreak some things broken by clang-format.

[jlpeyton]

Adding openmp-commits

[jlpeyton]

LGTM

runtime/src/kmp_affinity.cpp

for (i = 1; i < num_avail; i++) {
     // Find the most significant index whose id differs from the id for the
     // previous OS proc.
     for (index = maxIndex; index >= threadIdIndex; index--) {
       if (assign_thread_ids && (index == threadIdIndex)) {
         // Auto-assign the thread id field if it wasn't specified.
         if (threadInfo[i][threadIdIndex] == UINT_MAX) {
           threadInfo[i][threadIdIndex] = threadIdCt++;
         }
-        // Aparrently the thread id field was specified for some entries and not
+        // Apparently the thread id field was specified for some entries and not
         // others. Start the thread id counter off at the next higher thread id.
         else if (threadIdCt <= threadInfo[i][threadIdIndex]) {
           threadIdCt = threadInfo[i][threadIdIndex] + 1;
         }
       }
       if (threadInfo[i][index] != lastId[index]) {
         // Run through all indices which are less significant, and reset the
         // counts to 1. At all levels up to and including index, we need to

runtime/src/kmp_barrier.cpp

for (i = 1; i < team->t.t_nproc; ++i) {
       KMP_DEBUG_ASSERT(
           (TCR_4(other_threads[i]->th.th_bar[bs_forkjoin_barrier].bb.b_go) &
            ~(KMP_BARRIER_SLEEP_STATE)) == KMP_INIT_BARRIER_STATE);
       KMP_DEBUG_ASSERT(other_threads[i]->th.th_team == team);
     }
 #endif
 
     if (__kmp_tasking_mode != tskm_immediate_exec) {
-      __kmp_task_team_setup(
-          this_thr, team,
-          0); // 0 indicates setup current task team if nthreads > 1
+      // 0 indicates setup current task team if nthreads > 1
+      __kmp_task_team_setup(this_thr, team, 0);
     }
 
     /* The master thread may have changed its blocktime between the join barrier
        and the fork barrier. Copy the blocktime info to the thread, where
        __kmp_wait_template() can access it when the team struct is not
        guaranteed to exist. */
     // See note about the corresponding code in __kmp_join_barrier() being
     // performance-critical

runtime/src/kmp_runtime.cpp

 #endif
 static void __kmp_unregister_library(void); // called by __kmp_internal_end()
 static void __kmp_reap_thread(kmp_info_t *thread, int is_root);
 static kmp_info_t *__kmp_thread_pool_insert_pt = NULL;
 
 /* Calculate the identifier of the current thread */
 /* fast (and somewhat portable) way to get unique identifier of executing
    thread. Returns KMP_GTID_DNE if we haven't been assigned a gtid. */
-
 int __kmp_get_global_thread_id() {
   int i;
   kmp_info_t **other_threads;
   size_t stack_data;
   char *stack_addr;
   size_t stack_size;
   char *stack_base;
 
if (__kmp_nth + new_nthreads -
     }
     KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_ALL_THREADS reduced "
                   "reservation to %d threads\n",
                   master_tid, tl_nthreads));
     new_nthreads = tl_nthreads;
   }
 
   // Check if the threads array is large enough, or needs expanding.
-  //
   // See comment in __kmp_register_root() about the adjustment if
   // __kmp_threads[0] == NULL.
   capacity = __kmp_threads_capacity;
   if (TCR_PTR(__kmp_threads[0]) == NULL) {
     --capacity;
   }
   if (__kmp_nth + new_nthreads -
           (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
if (slotsAdded < slotsRequired) {
           __kmp_msg(kmp_ms_warning,
                     KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads),
                     KMP_HNT(SystemLimitOnThreads), __kmp_msg_null);
         }
       }
     }
   }
 
+#ifdef KMP_DEBUG
   if (new_nthreads == 1) {
     KC_TRACE(10,
              ("__kmp_reserve_threads: T#%d serializing team after reclaiming "
               "dead roots and rechecking; requested %d threads\n",
               __kmp_get_gtid(), set_nthreads));
-    return 1;
-  }
-
-  KC_TRACE(10, ("__kmp_reserve_threads: T#%d allocating %d threads; requested "
-                "%d threads\n",
+  } else {
+    KC_TRACE(10, ("__kmp_reserve_threads: T#%d allocating %d threads; requested"
+                  " %d threads\n",
                   __kmp_get_gtid(), new_nthreads, set_nthreads));
+  }
+#endif // KMP_DEBUG
   return new_nthreads;
 }
 
 /* Allocate threads from the thread pool and assign them to the new team. We are
    assured that there are enough threads available, because we checked on that
    earlier within critical section forkjoin */
 static void __kmp_fork_team_threads(kmp_root_t *root, kmp_team_t *team,
                                     kmp_info_t *master_th, int master_gtid) {
#if OMPT_SUPPORT
     }
 #endif
 
     // Nested level will be an index in the nested nthreads array
     level = parent_team->t.t_level;
     // used to launch non-serial teams even if nested is not allowed
     active_level = parent_team->t.t_active_level;
 #if OMP_40_ENABLED
-    teams_level =
-        master_th->th
-            .th_teams_level; // needed to check nesting inside the teams
+    // needed to check nesting inside the teams
+    teams_level = master_th->th.th_teams_level;
 #endif
 #if KMP_NESTED_HOT_TEAMS
     p_hot_teams = &master_th->th.th_hot_teams;
     if (*p_hot_teams == NULL && __kmp_hot_teams_max_level > 0) {
       *p_hot_teams = (kmp_hot_team_ptr_t *)__kmp_allocate(
           sizeof(kmp_hot_team_ptr_t) * __kmp_hot_teams_max_level);
       (*p_hot_teams)[0].hot_team = root->r.r_hot_team;
-      (*p_hot_teams)[0].hot_team_nth =
-          1; // it is either actual or not needed (when active_level > 0)
+      // it is either actual or not needed (when active_level > 0)
+      (*p_hot_teams)[0].hot_team_nth = 1;
     }
 #endif
 
 #if OMPT_SUPPORT
     if (ompt_enabled &&
         ompt_callbacks.ompt_callback(ompt_event_parallel_begin)) {
       int team_size = master_set_numthreads;
 
#endif
           // Keep extra threads hot in the team for possible next parallels
         }
         master_th->th.th_set_nproc = 0;
       }
 
 #if USE_DEBUGGER
       if (__kmp_debugging) { // Let debugger override number of threads.
         int nth = __kmp_omp_num_threads(loc);
-        if (nth >
-            0) { // 0 means debugger does not want to change number of threads.
+        if (nth > 0) { // 0 means debugger doesn't want to change num threads
           master_set_numthreads = nth;
         }; // if
       }; // if
 #endif
 
       KF_TRACE(10, ("__kmp_fork_call: before internal fork: root=%p, team=%p, "
                     "master_th=%p, gtid=%d\n",
                     root, parent_team, master_th, gtid));
#endif
       // __kmp_reserve_threads() to speedup nested serialized parallels.
       if (nthreads > 1) {
         if ((!get__nested(master_th) && (root->r.r_in_parallel
 #if OMP_40_ENABLED
                                          && !enter_teams
 #endif /* OMP_40_ENABLED */
                                          )) ||
             (__kmp_library == library_serial)) {
-          KC_TRACE(
-              10,
-              ("__kmp_fork_call: T#%d serializing team; requested %d threads\n",
+          KC_TRACE(10, ("__kmp_fork_call: T#%d serializing team; requested %d"
+                        " threads\n",
                         gtid, nthreads));
           nthreads = 1;
         }
       }
       if (nthreads > 1) {
         /* determine how many new threads we can use */
         __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
-
         nthreads = __kmp_reserve_threads(
             root, parent_team, master_tid, nthreads
 #if OMP_40_ENABLED
             /* AC: If we execute teams from parallel region (on host), then
                teams should be created but each can only have 1 thread if
                nesting is disabled. If teams called from serial region, then
                teams and their threads should be created regardless of the
                nesting setting. */
#endif
           __kmp_free(old_stack);
         }
         // Store master's task_state on stack
         master_th->th
             .th_task_state_memo_stack[master_th->th.th_task_state_top] =
             master_th->th.th_task_state;
         master_th->th.th_task_state_top++;
 #if KMP_NESTED_HOT_TEAMS
-        if (team ==
-            master_th->th.th_hot_teams[active_level]
-                .hot_team) { // Restore master's nested state if nested hot team
+        if (team == master_th->th.th_hot_teams[active_level].hot_team) {
+          // Restore master's nested state if nested hot team
           master_th->th.th_task_state =
               master_th->th
                   .th_task_state_memo_stack[master_th->th.th_task_state_top];
         } else {
 #endif
           master_th->th.th_task_state = 0;
 #if KMP_NESTED_HOT_TEAMS
         }
#if USE_ITT_NOTIFY
            __kmp_forkjoin_frames_mode == 1)) {
         kmp_uint64 tmp_time = 0;
         if (__itt_get_timestamp_ptr)
           tmp_time = __itt_get_timestamp();
         // Internal fork - report frame begin
         master_th->th.th_frame_time = tmp_time;
         if (__kmp_forkjoin_frames_mode == 3)
           team->t.t_region_time = tmp_time;
-      } else // only one notification scheme (either "submit" or
-// "forking/joined", not both)
+      } else
+// only one notification scheme (either "submit" or "forking/joined", not both)
 #endif /* USE_ITT_NOTIFY */
           if ((__itt_frame_begin_v3_ptr || KMP_ITT_DEBUG) &&
               __kmp_forkjoin_frames && !__kmp_forkjoin_frames_mode) {
         // Mark start of "parallel" region for VTune.
         __kmp_itt_region_forking(gtid, team->t.t_nproc, 0);
       }
     }
 #endif /* USE_ITT_BUILD */
#if USE_ITT_BUILD
     if (__itt_stack_caller_create_ptr) {
       team->t.t_stack_id =
           __kmp_itt_stack_caller_create(); // create new stack stitching id
       // before entering fork barrier
     }
 #endif /* USE_ITT_BUILD */
 
 #if OMP_40_ENABLED
-    if (ap) // AC: skip __kmp_internal_fork at teams construct, let only master
+    // AC: skip __kmp_internal_fork at teams construct, let only master
     // threads execute
+    if (ap)
 #endif /* OMP_40_ENABLED */
     {
       __kmp_internal_fork(loc, gtid, team);
       KF_TRACE(10, ("__kmp_internal_fork : after : root=%p, team=%p, "
                     "master_th=%p, gtid=%d\n",
                     root, team, master_th, gtid));
     }
 
static void __kmp_reinitialize_team(kmp_team_t *team,
                                     ident_t *loc) {
   KF_TRACE(10, ("__kmp_reinitialize_team: enter this_thread=%p team=%p\n",
                 team->t.t_threads[0], team));
   KMP_DEBUG_ASSERT(team && new_icvs);
   KMP_DEBUG_ASSERT((!TCR_4(__kmp_init_parallel)) || new_icvs->nproc);
   KMP_CHECK_UPDATE(team->t.t_ident, loc);
 
   KMP_CHECK_UPDATE(team->t.t_id, KMP_GEN_TEAM_ID());
-
   // Copy ICVs to the master thread's implicit taskdata
   __kmp_init_implicit_task(loc, team->t.t_threads[0], team, 0, FALSE);
   copy_icvs(&team->t.t_implicit_task_taskdata[0].td_icvs, new_icvs);
 
   KF_TRACE(10, ("__kmp_reinitialize_team: exit this_thread=%p team=%p\n",
                 team->t.t_threads[0], team));
 }
 

 #endif
 
     // Has the number of threads changed?
     /* Let's assume the most common case is that the number of threads is
        unchanged, and put that case first. */
     if (team->t.t_nproc == new_nproc) { // Check changes in number of threads
       KA_TRACE(20, ("__kmp_allocate_team: reusing hot team\n"));
       // This case can mean that omp_set_num_threads() was called and the hot
-      // team size
-      // was already reduced, so we check the special flag
+      // team size was already reduced, so we check the special flag
       if (team->t.t_size_changed == -1) {
         team->t.t_size_changed = 1;
       } else {
         KMP_CHECK_UPDATE(team->t.t_size_changed, 0);
       }
 
       // TODO???: team->t.t_max_active_levels = new_max_active_levels;
       kmp_r_sched_t new_sched = new_icvs->sched;

runtime/src/kmp_tasking.cpp

#endif
   task->td_flags.freed = 0;
 
 #if OMP_40_ENABLED
   task->td_depnode = NULL;
 #endif
 
   if (set_curr_task) { // only do this init first time thread is created
     task->td_incomplete_child_tasks = 0;
-    task->td_allocated_child_tasks = 0; // Not used: don't need to
-// deallocate implicit task
+    // Not used: don't need to deallocate implicit task
+    task->td_allocated_child_tasks = 0;
 #if OMP_40_ENABLED
     task->td_taskgroup = NULL; // An implicit task does not have taskgroup
     task->td_dephash = NULL;
 #endif
     __kmp_push_current_task_to_thread(this_thr, team, tid);
   } else {
     KMP_DEBUG_ASSERT(task->td_incomplete_child_tasks == 0);
     KMP_DEBUG_ASSERT(task->td_allocated_child_tasks == 0);