Update stats-gathering code

[jlpeyton]

Summary

Have developer timers use partitioning scheme which also required that some redundant developer timers be removed in favor of the already existing normal timers. Move per thread stats initialization to just after global thread id assignment which is as early as possible. Also put all global stats initialization code in __kmp_stats_init() and all global stats destruction code in __kmp_stats_fini().

[tlwilmar]

LGTM

runtime/src/kmp_barrier.cpp

 // ---------------------------- Barrier Algorithms ----------------------------
 
 // Linear Barrier
 static void
 __kmp_linear_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
                             void (*reduce)(void *, void *)
                             USE_ITT_BUILD_ARG(void * itt_sync_obj) )
 {
-    KMP_TIME_DEVELOPER_BLOCK(KMP_linear_gather);
+    KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_linear_gather);
     register kmp_team_t *team = this_thr->th.th_team;
     register kmp_bstate_t *thr_bar = & this_thr->th.th_bar[bt].bb;
     register kmp_info_t **other_threads = team->t.t_threads;
 
     KA_TRACE(20, ("__kmp_linear_barrier_gather: T#%d(%d:%d) enter for barrier type %d\n",
                   gtid, team->t.t_id, tid, bt));
     KMP_DEBUG_ASSERT(this_thr == other_threads[this_thr->th.th_info.ds.ds_tid]);
 
KA_TRACE(20, ("__kmp_linear_barrier_gather: T#%d(%d:%d) exit for barrier type %d\n",
                   gtid, team->t.t_id, tid, bt));
 }
 
 static void
 __kmp_linear_barrier_release(enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
                              int propagate_icvs
                              USE_ITT_BUILD_ARG(void *itt_sync_obj) )
 {
-    KMP_TIME_DEVELOPER_BLOCK(KMP_linear_release);
+    KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_linear_release);
     register kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
     register kmp_team_t *team;
 
     if (KMP_MASTER_TID(tid)) {
         register unsigned int i;
         register kmp_uint32 nproc = this_thr->th.th_team_nproc;
         register kmp_info_t **other_threads;
 
         team = __kmp_threads[gtid]->th.th_team;
         KMP_DEBUG_ASSERT(team != NULL);
         other_threads = team->t.t_threads;
 
         KA_TRACE(20, ("__kmp_linear_barrier_release: T#%d(%d:%d) master enter for barrier type %d\n",
                       gtid, team->t.t_id, tid, bt));
 
         if (nproc > 1) {
 #if KMP_BARRIER_ICV_PUSH
             {
-                KMP_TIME_DEVELOPER_BLOCK(USER_icv_copy);
+                KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_icv_copy);
                 if (propagate_icvs) {
                     ngo_load(&team->t.t_implicit_task_taskdata[0].td_icvs);
                     for (i=1; i<nproc; ++i) {
                         __kmp_init_implicit_task(team->t.t_ident, team->t.t_threads[i], team, i, FALSE);
                         ngo_store_icvs(&team->t.t_implicit_task_taskdata[i].td_icvs,
                                        &team->t.t_implicit_task_taskdata[0].td_icvs);
                     }
                     ngo_sync();

 }
 
 // Tree barrier
 static void
 __kmp_tree_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
                           void (*reduce)(void *, void *)
                           USE_ITT_BUILD_ARG(void *itt_sync_obj) )
 {
-    KMP_TIME_DEVELOPER_BLOCK(KMP_tree_gather);
+    KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_tree_gather);
     register kmp_team_t *team = this_thr->th.th_team;
     register kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
     register kmp_info_t **other_threads = team->t.t_threads;
     register kmp_uint32 nproc = this_thr->th.th_team_nproc;
     register kmp_uint32 branch_bits = __kmp_barrier_gather_branch_bits[bt];
     register kmp_uint32 branch_factor = 1 << branch_bits;
     register kmp_uint32 child;
     register kmp_uint32 child_tid;
KA_TRACE(20, ("__kmp_tree_barrier_gather: T#%d(%d:%d) exit for barrier type %d\n",
                   gtid, team->t.t_id, tid, bt));
 }
 
 static void
 __kmp_tree_barrier_release(enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
                            int propagate_icvs
                            USE_ITT_BUILD_ARG(void *itt_sync_obj) )
 {
-    KMP_TIME_DEVELOPER_BLOCK(KMP_tree_release);
+    KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_tree_release);
     register kmp_team_t *team;
     register kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
     register kmp_uint32 nproc;
     register kmp_uint32 branch_bits = __kmp_barrier_release_branch_bits[bt];
     register kmp_uint32 branch_factor = 1 << branch_bits;
     register kmp_uint32 child;
     register kmp_uint32 child_tid;
 

 #if KMP_CACHE_MANAGE
             // Prefetch next thread's go count
             if (child+1 <= branch_factor && child_tid+1 < nproc)
                 KMP_CACHE_PREFETCH(&other_threads[child_tid+1]->th.th_bar[bt].bb.b_go);
 #endif /* KMP_CACHE_MANAGE */
 
 #if KMP_BARRIER_ICV_PUSH
             {
-                KMP_TIME_DEVELOPER_BLOCK(USER_icv_copy);
+                KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_icv_copy);
                 if (propagate_icvs) {
                     __kmp_init_implicit_task(team->t.t_ident, team->t.t_threads[child_tid],
                                              team, child_tid, FALSE);
                     copy_icvs(&team->t.t_implicit_task_taskdata[child_tid].td_icvs,
                               &team->t.t_implicit_task_taskdata[0].td_icvs);
                 }
             }
 #endif // KMP_BARRIER_ICV_PUSH

 
 
 // Hyper Barrier
 static void
 __kmp_hyper_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
                            void (*reduce)(void *, void *)
                            USE_ITT_BUILD_ARG(void *itt_sync_obj) )
 {
-    KMP_TIME_DEVELOPER_BLOCK(KMP_hyper_gather);
+    KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_hyper_gather);
     register kmp_team_t *team = this_thr->th.th_team;
     register kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
     register kmp_info_t **other_threads = team->t.t_threads;
     register kmp_uint64 new_state = KMP_BARRIER_UNUSED_STATE;
     register kmp_uint32 num_threads = this_thr->th.th_team_nproc;
     register kmp_uint32 branch_bits = __kmp_barrier_gather_branch_bits[bt];
     register kmp_uint32 branch_factor = 1 << branch_bits;
     register kmp_uint32 offset;

 
 // The reverse versions seem to beat the forward versions overall
 #define KMP_REVERSE_HYPER_BAR
 static void
 __kmp_hyper_barrier_release(enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
                             int propagate_icvs
                             USE_ITT_BUILD_ARG(void *itt_sync_obj) )
 {
-    KMP_TIME_DEVELOPER_BLOCK(KMP_hyper_release);
+    KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_hyper_release);
     register kmp_team_t    *team;
     register kmp_bstate_t  *thr_bar       = & this_thr -> th.th_bar[ bt ].bb;
     register kmp_info_t   **other_threads;
     register kmp_uint32     num_threads;
     register kmp_uint32     branch_bits   = __kmp_barrier_release_branch_bits[ bt ];
     register kmp_uint32     branch_factor = 1 << branch_bits;
     register kmp_uint32     child;
     register kmp_uint32     child_tid;
__kmp_init_hierarchical_barrier_thread(enum barrier_type bt, kmp_bstate_t *thr_bar, kmp_uint32 nproc,
     return retval;
 }
 
 static void
 __kmp_hierarchical_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr,
                                   int gtid, int tid, void (*reduce) (void *, void *)
                                   USE_ITT_BUILD_ARG(void * itt_sync_obj) )
 {
-    KMP_TIME_DEVELOPER_BLOCK(KMP_hier_gather);
+    KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_hier_gather);
     register kmp_team_t *team = this_thr->th.th_team;
     register kmp_bstate_t *thr_bar = & this_thr->th.th_bar[bt].bb;
     register kmp_uint32 nproc = this_thr->th.th_team_nproc;
     register kmp_info_t **other_threads = team->t.t_threads;
     register kmp_uint64 new_state;
 
     int level = team->t.t_level;
 #if OMP_40_ENABLED
KA_TRACE(20, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) exit for barrier type %d\n",
                   gtid, team->t.t_id, tid, bt));
 }
 
 static void
 __kmp_hierarchical_barrier_release(enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
                                    int propagate_icvs
                                    USE_ITT_BUILD_ARG(void * itt_sync_obj) )
 {
-    KMP_TIME_DEVELOPER_BLOCK(KMP_hier_release);
+    KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_hier_release);
     register kmp_team_t *team;
     register kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
     register kmp_uint32 nproc;
     bool team_change = false; // indicates on-core barrier shouldn't be used
 
     if (KMP_MASTER_TID(tid)) {
         team = __kmp_threads[gtid]->th.th_team;
         KMP_DEBUG_ASSERT(team != NULL);

 // Internal function to do a barrier.
 /* If is_split is true, do a split barrier, otherwise, do a plain barrier
    If reduce is non-NULL, do a split reduction barrier, otherwise, do a split barrier
    Returns 0 if master thread, 1 if worker thread.  */
 int
 __kmp_barrier(enum barrier_type bt, int gtid, int is_split, size_t reduce_size,
               void *reduce_data, void (*reduce)(void *, void *))
 {
-    KMP_TIME_DEVELOPER_BLOCK(KMP_barrier);
-    KMP_SET_THREAD_STATE_BLOCK(PLAIN_BARRIER);
     KMP_TIME_PARTITIONED_BLOCK(OMP_plain_barrier);
+    KMP_SET_THREAD_STATE_BLOCK(PLAIN_BARRIER);
     register int tid = __kmp_tid_from_gtid(gtid);
     register kmp_info_t *this_thr = __kmp_threads[gtid];
     register kmp_team_t *team = this_thr->th.th_team;
     register int status = 0;
     ident_t *loc = __kmp_threads[gtid]->th.th_ident;
 #if OMPT_SUPPORT
     ompt_task_id_t my_task_id;
     ompt_parallel_id_t my_parallel_id;
#endif
 
     return status;
 }
 
 
 void
 __kmp_end_split_barrier(enum barrier_type bt, int gtid)
 {
-    KMP_TIME_DEVELOPER_BLOCK(KMP_end_split_barrier);
+    KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_end_split_barrier);
+    KMP_SET_THREAD_STATE_BLOCK(PLAIN_BARRIER);
     int tid = __kmp_tid_from_gtid(gtid);
     kmp_info_t *this_thr = __kmp_threads[gtid];
     kmp_team_t *team = this_thr->th.th_team;
 
     ANNOTATE_NEW_BARRIER_BEGIN(&team->t.t_bar);
     if (!team->t.t_serialized) {
         if (KMP_MASTER_GTID(gtid)) {
             switch (__kmp_barrier_release_pattern[bt]) {
__kmp_end_split_barrier(enum barrier_type bt, int gtid)
     }
     ANNOTATE_NEW_BARRIER_END(&team->t.t_bar);
 }
 
 
 void
 __kmp_join_barrier(int gtid)
 {
-    KMP_TIME_PARTITIONED_BLOCK(OMP_fork_join_barrier);
+    KMP_TIME_PARTITIONED_BLOCK(OMP_join_barrier);
     KMP_SET_THREAD_STATE_BLOCK(FORK_JOIN_BARRIER);
-    KMP_TIME_DEVELOPER_BLOCK(KMP_join_barrier);
     register kmp_info_t *this_thr = __kmp_threads[gtid];
     register kmp_team_t *team;
     register kmp_uint nproc;
     kmp_info_t *master_thread;
     int tid;
 #ifdef KMP_DEBUG
     int team_id;
 #endif /* KMP_DEBUG */
#endif
     ANNOTATE_NEW_BARRIER_END(&team->t.t_bar);
 }
 
 
 // TODO release worker threads' fork barriers as we are ready instead of all at once
 void
 __kmp_fork_barrier(int gtid, int tid)
 {
-    KMP_TIME_PARTITIONED_BLOCK(OMP_fork_join_barrier);
+    KMP_TIME_PARTITIONED_BLOCK(OMP_fork_barrier);
     KMP_SET_THREAD_STATE_BLOCK(FORK_JOIN_BARRIER);
-    KMP_TIME_DEVELOPER_BLOCK(KMP_fork_barrier);
     kmp_info_t *this_thr = __kmp_threads[gtid];
     kmp_team_t *team = (tid == 0) ? this_thr->th.th_team : NULL;
 #if USE_ITT_BUILD
     void * itt_sync_obj = NULL;
 #endif /* USE_ITT_BUILD */
     if (team)
       ANNOTATE_NEW_BARRIER_END(&team->t.t_bar);
 

 
 #if KMP_BARRIER_ICV_PULL
     /* Master thread's copy of the ICVs was set up on the implicit taskdata in
        __kmp_reinitialize_team. __kmp_fork_call() assumes the master thread's implicit task has
        this data before this function is called. We cannot modify __kmp_fork_call() to look at
        the fixed ICVs in the master's thread struct, because it is not always the case that the
        threads arrays have been allocated when __kmp_fork_call() is executed. */
     {
-        KMP_TIME_DEVELOPER_BLOCK(USER_icv_copy);
+        KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_icv_copy);
         if (!KMP_MASTER_TID(tid)) {  // master thread already has ICVs
             // Copy the initial ICVs from the master's thread struct to the implicit task for this tid.
             KA_TRACE(10, ("__kmp_fork_barrier: T#%d(%d) is PULLing ICVs\n", gtid, tid));
             __kmp_init_implicit_task(team->t.t_ident, team->t.t_threads[tid], team, tid, FALSE);
             copy_icvs(&team->t.t_implicit_task_taskdata[tid].td_icvs,
                       &team->t.t_threads[0]->th.th_bar[bs_forkjoin_barrier].bb.th_fixed_icvs);
         }
     }
#endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */
     ANNOTATE_NEW_BARRIER_END(&team->t.t_bar);
     KA_TRACE(10, ("__kmp_fork_barrier: T#%d(%d:%d) is leaving\n", gtid, team->t.t_id, tid));
 }
 
 
 void
 __kmp_setup_icv_copy(kmp_team_t *team, int new_nproc, kmp_internal_control_t *new_icvs, ident_t *loc )
 {
-    KMP_TIME_DEVELOPER_BLOCK(KMP_setup_icv_copy);
+    KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_setup_icv_copy);
 
     KMP_DEBUG_ASSERT(team && new_nproc && new_icvs);
     KMP_DEBUG_ASSERT((!TCR_4(__kmp_init_parallel)) || new_icvs->nproc);
 
     /* Master thread's copy of the ICVs was set up on the implicit taskdata in
        __kmp_reinitialize_team. __kmp_fork_call() assumes the master thread's implicit task has
        this data before this function is called. */
 #if KMP_BARRIER_ICV_PULL

runtime/src/kmp_global.c

 #endif
 
 #if KMP_STATS_ENABLED
 #include "kmp_stats.h"
 // lock for modifying the global __kmp_stats_list
 kmp_tas_lock_t __kmp_stats_lock;
 
 // global list of per thread stats, the head is a sentinel node which accumulates all stats produced before __kmp_create_worker is called.
-kmp_stats_list __kmp_stats_list;
+kmp_stats_list* __kmp_stats_list;
 
 // thread local pointer to stats node within list
-__thread kmp_stats_list* __kmp_stats_thread_ptr = &__kmp_stats_list;
+__thread kmp_stats_list* __kmp_stats_thread_ptr = NULL;
 
 // gives reference tick for all events (considered the 0 tick)
 tsc_tick_count __kmp_stats_start_time;
 #endif
 
 /* ----------------------------------------------------- */
 /* INITIALIZATION VARIABLES */
 /* they are syncronized to write during init, but read anytime */

runtime/src/kmp_runtime.c

 #if OMP_40_ENABLED
     int             active_level;
     int             teams_level;
 #endif
 #if KMP_NESTED_HOT_TEAMS
     kmp_hot_team_ptr_t **p_hot_teams;
 #endif
     { // KMP_TIME_BLOCK
-    KMP_TIME_DEVELOPER_BLOCK(KMP_fork_call);
+    KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_fork_call);
     KMP_COUNT_VALUE(OMP_PARALLEL_args, argc);
 
     KA_TRACE( 20, ("__kmp_fork_call: enter T#%d\n", gtid ));
     if ( __kmp_stkpadding > 0 &&  __kmp_root[gtid] != NULL ) {
         /* Some systems prefer the stack for the root thread(s) to start with */
         /* some gap from the parent stack to prevent false sharing. */
         void *dummy = KMP_ALLOCA(__kmp_stkpadding);
         /* These 2 lines below are so this does not get optimized out */
#endif /* OMP_40_ENABLED */
     /* Invoke microtask for MASTER thread */
     KA_TRACE( 20, ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n",
                 gtid, team->t.t_id, team->t.t_pkfn ) );
     }  // END of timer KMP_fork_call block
 
     {
         KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
         KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
-        // KMP_TIME_DEVELOPER_BLOCK(USER_master_invoke);
         if (! team->t.t_invoke( gtid )) {
             KMP_ASSERT2( 0, "cannot invoke microtask for MASTER thread" );
         }
     }
     KA_TRACE( 20, ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n",
         gtid, team->t.t_id, team->t.t_pkfn ) );
     KMP_MB();       /* Flush all pending memory write invalidates.  */
 

 #if OMPT_SUPPORT
                , enum fork_context_e fork_context
 #endif
 #if OMP_40_ENABLED
                , int exit_teams
 #endif /* OMP_40_ENABLED */
 )
 {
-    KMP_TIME_DEVELOPER_BLOCK(KMP_join_call);
+    KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_join_call);
     kmp_team_t     *team;
     kmp_team_t     *parent_team;
     kmp_info_t     *master_th;
     kmp_root_t     *root;
     int             master_active;
     int             i;
 
     KA_TRACE( 20, ("__kmp_join_call: enter T#%d\n", gtid ));

 #endif /* KMP_ADJUST_BLOCKTIME */
 
     /* setup this new hierarchy */
     if( ! ( root = __kmp_root[gtid] )) {
         root = __kmp_root[gtid] = (kmp_root_t*) __kmp_allocate( sizeof(kmp_root_t) );
         KMP_DEBUG_ASSERT( ! root->r.r_root_team );
     }
 
+#if KMP_STATS_ENABLED
+    // Initialize stats as soon as possible (right after gtid assignment).
+    __kmp_stats_thread_ptr = __kmp_stats_list->push_back(gtid);
+    KMP_START_EXPLICIT_TIMER(OMP_worker_thread_life);
+    KMP_SET_THREAD_STATE(SERIAL_REGION);
+    KMP_INIT_PARTITIONED_TIMERS(OMP_serial);
+#endif
     __kmp_initialize_root( root );
 
     /* setup new root thread structure */
     if( root->r.r_uber_thread ) {
         root_thread = root->r.r_uber_thread;
     } else {
         root_thread = (kmp_info_t*) __kmp_allocate( sizeof(kmp_info_t) );
         if ( __kmp_storage_map ) {
#if OMPT_SUPPORT
     ompt_parallel_id_t ompt_parallel_id,
 #endif
 #if OMP_40_ENABLED
     kmp_proc_bind_t new_proc_bind,
 #endif
     kmp_internal_control_t *new_icvs,
     int argc USE_NESTED_HOT_ARG(kmp_info_t *master) )
 {
-    KMP_TIME_DEVELOPER_BLOCK(KMP_allocate_team);
+    KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_allocate_team);
     int f;
     kmp_team_t *team;
     int use_hot_team = ! root->r.r_active;
     int level = 0;
 
     KA_TRACE( 20, ("__kmp_allocate_team: called\n"));
     KMP_DEBUG_ASSERT( new_nproc >=1 && argc >=0 );
     KMP_DEBUG_ASSERT( max_nproc >= new_nproc );
#if OMPT_SUPPORT
                 if (ompt_enabled) {
                     this_thr->th.ompt_thread_info.state = ompt_state_work_parallel;
                     // Initialize OMPT task id for implicit task.
                     int tid = __kmp_tid_from_gtid(gtid);
                     task_info->task_id = __ompt_task_id_new(tid);
                 }
 #endif
 
-                KMP_STOP_DEVELOPER_EXPLICIT_TIMER(USER_launch_thread_loop);
                 {
-                    KMP_TIME_DEVELOPER_BLOCK(USER_worker_invoke);
                     KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
                     KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
                     rc = (*pteam)->t.t_invoke( gtid );
                 }
-                KMP_START_DEVELOPER_EXPLICIT_TIMER(USER_launch_thread_loop);
                 KMP_ASSERT( rc );
 
 #if OMPT_SUPPORT
                 if (ompt_enabled) {
                     /* no frame set while outside task */
                     task_info->frame.exit_runtime_frame = NULL;
 
                     this_thr->th.ompt_thread_info.state = ompt_state_overhead;
#endif
 
     /* initialize the locks */
 #if KMP_USE_ADAPTIVE_LOCKS
 #if KMP_DEBUG_ADAPTIVE_LOCKS
     __kmp_init_speculative_stats();
 #endif
 #endif
 #if KMP_STATS_ENABLED
-    __kmp_init_tas_lock( & __kmp_stats_lock );
+    __kmp_stats_init();
 #endif
     __kmp_init_lock( & __kmp_global_lock     );
     __kmp_init_queuing_lock( & __kmp_dispatch_lock );
     __kmp_init_lock( & __kmp_debug_lock      );
     __kmp_init_atomic_lock( & __kmp_atomic_lock     );
     __kmp_init_atomic_lock( & __kmp_atomic_lock_1i  );
     __kmp_init_atomic_lock( & __kmp_atomic_lock_2i  );
     __kmp_init_atomic_lock( & __kmp_atomic_lock_4i  );
#endif
     KMP_INTERNAL_FREE( __kmp_nested_nth.nth );
     __kmp_nested_nth.nth = NULL;
     __kmp_nested_nth.size = 0;
     __kmp_nested_nth.used = 0;
 
     __kmp_i18n_catclose();
 
 #if KMP_STATS_ENABLED
-    __kmp_accumulate_stats_at_exit();
-    __kmp_stats_list.deallocate();
+    __kmp_stats_fini();
 #endif
 
     KA_TRACE( 10, ("__kmp_cleanup: exit\n" ) );
 }
 
 /* ------------------------------------------------------------------------ */
 /* ------------------------------------------------------------------------ */
 

runtime/src/kmp_stats.h

#define KMP_FOREACH_COUNTER(macro, arg)                         \
     macro (OMP_set_lock, 0, arg)                                \
     macro (OMP_test_lock, 0, arg)                               \
     macro (REDUCE_wait, 0, arg)                                 \
     macro (REDUCE_nowait, 0, arg)                               \
     macro (OMP_TASKYIELD, 0, arg)                               \
     macro (OMP_TASKLOOP, 0, arg)                                \
     macro (TASK_executed, 0, arg)                               \
     macro (TASK_cancelled, 0, arg)                              \
-    macro (TASK_stolen, 0, arg)                                 \
-    macro (LAST,0,arg)
+    macro (TASK_stolen, 0, arg)
 
 /*!
  * \brief Add new timers under KMP_FOREACH_TIMER() macro in kmp_stats.h
  *
  * @param macro a user defined macro that takes three arguments - macro(TIMER_NAME, flags, arg)
  * @param arg a user defined argument to send to the user defined macro
  *
  * \details A timer collects multiple samples of some count in each thread and then finally aggregates alll of the samples from all of the threads.
  * For most timers the printing code also provides an aggregation over the thread totals. These are printed as TOTAL_foo.
  * The count is normally a time (in ticks), hence the name "timer". (But can be any value, so we use this for "number of arguments passed to fork"
  * as well).
  * For timers the threads are not significant, it's the individual observations that count, so the statistics are at that level.
  * Format is "macro(name, flags, arg)"
  *
  * @ingroup STATS_GATHERING2
  */
 #define KMP_FOREACH_TIMER(macro, arg)                              \
-    macro (OMP_worker_thread_life, 0, arg)                         \
+    macro (OMP_worker_thread_life, stats_flags_e::logEvent, arg)   \
     macro (FOR_static_scheduling, 0, arg)                          \
     macro (FOR_dynamic_scheduling, 0, arg)                         \
     macro (OMP_critical,  0, arg)                                  \
     macro (OMP_critical_wait,  0, arg)                             \
     macro (OMP_single,    0, arg)                                  \
     macro (OMP_master,    0, arg)                                  \
-    macro (OMP_idle, 0, arg)                                       \
-    macro (OMP_plain_barrier, 0, arg)                              \
-    macro (OMP_fork_join_barrier, 0, arg)                          \
-    macro (OMP_parallel, 0, arg)                                   \
+    macro (OMP_idle, stats_flags_e::logEvent, arg)                 \
+    macro (OMP_plain_barrier, stats_flags_e::logEvent, arg)        \
+    macro (OMP_fork_barrier, stats_flags_e::logEvent, arg)         \
+    macro (OMP_join_barrier, stats_flags_e::logEvent, arg)         \
+    macro (OMP_parallel, stats_flags_e::logEvent, arg)             \
     macro (OMP_task_immediate, 0, arg)                             \
     macro (OMP_task_taskwait, 0, arg)                              \
     macro (OMP_task_taskyield, 0, arg)                             \
     macro (OMP_task_taskgroup, 0, arg)                             \
     macro (OMP_task_join_bar, 0, arg)                              \
     macro (OMP_task_plain_bar, 0, arg)                             \
-    macro (OMP_serial, 0, arg)                                     \
+    macro (OMP_serial, stats_flags_e::logEvent, arg)               \
     macro (OMP_taskloop_scheduling, 0, arg)                        \
     macro (OMP_set_numthreads,    stats_flags_e::noUnits | stats_flags_e::noTotal, arg) \
     macro (OMP_PARALLEL_args,     stats_flags_e::noUnits | stats_flags_e::noTotal, arg) \
     macro (FOR_static_iterations, stats_flags_e::noUnits | stats_flags_e::noTotal, arg) \
     macro (FOR_dynamic_iterations,stats_flags_e::noUnits | stats_flags_e::noTotal, arg) \
-    KMP_FOREACH_DEVELOPER_TIMER(macro, arg)                             \
-    macro (LAST,0, arg)
+    KMP_FOREACH_DEVELOPER_TIMER(macro, arg)
 
 
 // OMP_start_end          -- Time from when OpenMP is initialized until the stats are printed at exit
 // OMP_serial             -- Thread zero time executing serial code
 // OMP_work               -- Elapsed time in code dispatched by a fork (measured in the thread)
 // OMP_barrier            -- Time at "real" barriers (includes task time)
 // FOR_static_scheduling  -- Time spent doing scheduling for a static "for"
 // FOR_dynamic_scheduling -- Time spent doing scheduling for a dynamic "for"

 // KMP_setup_icv_copy     -- time in __kmp_setup_icv_copy
 // KMP_icv_copy           -- start/stop timer for any ICV copying
 // KMP_linear_gather      -- time in __kmp_linear_barrier_gather
 // KMP_linear_release     -- time in __kmp_linear_barrier_release
 // KMP_tree_gather        -- time in __kmp_tree_barrier_gather
 // KMP_tree_release       -- time in __kmp_tree_barrier_release
 // KMP_hyper_gather       -- time in __kmp_hyper_barrier_gather
 // KMP_hyper_release      -- time in __kmp_hyper_barrier_release
-# define KMP_FOREACH_DEVELOPER_TIMER(macro, arg)                        \
-    macro (KMP_fork_call, 0, arg)                                       \
-    macro (KMP_join_call, 0, arg)                                       \
-    macro (KMP_fork_barrier, stats_flags_e::logEvent, arg)              \
-    macro (KMP_join_barrier, stats_flags_e::logEvent, arg)              \
-    macro (KMP_barrier, 0, arg)                                         \
-    macro (KMP_end_split_barrier, 0, arg)                               \
-    macro (KMP_hier_gather, 0, arg)                                     \
-    macro (KMP_hier_release, 0, arg)                                    \
-    macro (KMP_hyper_gather,  stats_flags_e::logEvent, arg)             \
-    macro (KMP_hyper_release,  stats_flags_e::logEvent, arg)            \
-    macro (KMP_linear_gather, 0, arg)                                   \
-    macro (KMP_linear_release, 0, arg)                                  \
-    macro (KMP_tree_gather, 0, arg)                                     \
-    macro (KMP_tree_release, 0, arg)                                    \
-    macro (USER_master_invoke, stats_flags_e::logEvent, arg)            \
-    macro (USER_worker_invoke, stats_flags_e::logEvent, arg)            \
-    macro (USER_resume, stats_flags_e::logEvent, arg)                   \
-    macro (USER_suspend, stats_flags_e::logEvent, arg)                  \
-    macro (USER_launch_thread_loop, stats_flags_e::logEvent, arg)       \
-    macro (KMP_allocate_team, 0, arg)                                   \
-    macro (KMP_setup_icv_copy, 0, arg)                                  \
+# define KMP_FOREACH_DEVELOPER_TIMER(macro, arg) \
+    macro (KMP_fork_call, 0, arg)                \
+    macro (KMP_join_call, 0, arg)                \
+    macro (KMP_end_split_barrier, 0, arg)        \
+    macro (KMP_hier_gather, 0, arg)              \
+    macro (KMP_hier_release, 0, arg)             \
+    macro (KMP_hyper_gather, 0, arg)             \
+    macro (KMP_hyper_release, 0, arg)            \
+    macro (KMP_linear_gather, 0, arg)            \
+    macro (KMP_linear_release, 0, arg)           \
+    macro (KMP_tree_gather, 0, arg)              \
+    macro (KMP_tree_release, 0, arg)             \
+    macro (USER_resume, 0, arg)                  \
+    macro (USER_suspend, 0, arg)                 \
+    macro (KMP_allocate_team, 0, arg)            \
+    macro (KMP_setup_icv_copy, 0, arg)           \
     macro (USER_icv_copy, 0, arg)
 #else
 # define KMP_FOREACH_DEVELOPER_TIMER(macro, arg)
 #endif
 
 /*!
  * \brief Add new explicit timers under KMP_FOREACH_EXPLICIT_TIMER() macro.
  *
  * @param macro a user defined macro that takes three arguments - macro(TIMER_NAME, flags, arg)
  * @param arg a user defined argument to send to the user defined macro
  *
  * \warning YOU MUST HAVE THE SAME NAMED TIMER UNDER KMP_FOREACH_TIMER() OR ELSE BAD THINGS WILL HAPPEN!
  *
  * \details Explicit timers are ones where we need to allocate a timer itself (as well as the accumulated timing statistics).
  * We allocate these on a per-thread basis, and explicitly start and stop them.
  * Block timers just allocate the timer itself on the stack, and use the destructor to notice block exit; they don't
  * need to be defined here.
  * The name here should be the same as that of a timer above.
  *
  * @ingroup STATS_GATHERING
 */
-#define KMP_FOREACH_EXPLICIT_TIMER(macro, arg)     \
-    macro(OMP_worker_thread_life, 0, arg)          \
-    macro(FOR_static_scheduling, 0, arg)           \
-    macro(FOR_dynamic_scheduling, 0, arg)          \
-    macro(OMP_critical, 0, arg)                    \
-    macro(OMP_critical_wait, 0, arg)               \
-    macro(OMP_single, 0, arg)                      \
-    macro(OMP_master, 0, arg)                      \
-    macro(OMP_idle, 0, arg)                        \
-    macro(OMP_plain_barrier, 0, arg)               \
-    macro(OMP_fork_join_barrier, 0, arg)           \
-    macro(OMP_parallel, 0, arg)                    \
-    macro(OMP_task_immediate, 0, arg)              \
-    macro(OMP_task_taskwait, 0, arg)               \
-    macro(OMP_task_taskyield, 0, arg)              \
-    macro(OMP_task_taskgroup, 0, arg)              \
-    macro(OMP_task_join_bar, 0, arg)               \
-    macro(OMP_task_plain_bar, 0, arg)              \
-    macro(OMP_serial, 0, arg)                      \
-    macro(OMP_taskloop_scheduling, 0, arg)         \
-    KMP_FOREACH_EXPLICIT_DEVELOPER_TIMER(macro,arg)     \
-    macro(LAST, 0, arg)
-
-#if (KMP_DEVELOPER_STATS)
-# define KMP_FOREACH_EXPLICIT_DEVELOPER_TIMER(macro, arg)               \
-    macro(USER_launch_thread_loop, stats_flags_e::logEvent, arg)
-#else
-# define KMP_FOREACH_EXPLICIT_DEVELOPER_TIMER(macro, arg)
-#endif
+#define KMP_FOREACH_EXPLICIT_TIMER(macro, arg) \
+    KMP_FOREACH_TIMER(macro, arg)
 
 #define ENUMERATE(name,ignore,prefix) prefix##name,
 enum timer_e {
     KMP_FOREACH_TIMER(ENUMERATE, TIMER_)
+    TIMER_LAST
 };
 
 enum explicit_timer_e {
     KMP_FOREACH_EXPLICIT_TIMER(ENUMERATE, EXPLICIT_TIMER_)
+    EXPLICIT_TIMER_LAST
 };
 
 enum counter_e {
     KMP_FOREACH_COUNTER(ENUMERATE, COUNTER_)
+    COUNTER_LAST
 };
 #undef ENUMERATE
 
 class timerPair {
     explicit_timer_e timer_index;
     timer_e timer;
  public:
     timerPair(explicit_timer_e ti, timer_e t) : timer_index(ti), timer(t) {}

  public:
     explicitTimer () : stat(0), startTime(0), pauseStartTime(0), totalPauseTime() { }
     explicitTimer (timeStat * s) : stat(s), startTime(), pauseStartTime(0), totalPauseTime() { }
 
     void setStat (timeStat *s) { stat = s; }
     void start(timer_e timerEnumValue);
     void pause() { pauseStartTime = tsc_tick_count::now(); }
     void resume() { totalPauseTime += (tsc_tick_count::now() - pauseStartTime); }
-    void stop(timer_e timerEnumValue);
+    void stop(timer_e timerEnumValue, kmp_stats_list* stats_ptr = nullptr);
     void reset() { startTime = 0; pauseStartTime = 0; totalPauseTime = 0; }
 };
 
 // Where all you need is to time a block, this is enough.
 // (It avoids the need to have an explicit end, leaving the scope suffices.)
 class blockTimer : public explicitTimer
 {
     timer_e timerEnumValue;
public:
     kmp_stats_output_module() { init(); }
     void outputStats(const char* heading);
 };
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 void __kmp_stats_init();
+void __kmp_stats_fini();
 void __kmp_reset_stats();
 void __kmp_output_stats(const char *);
 void __kmp_accumulate_stats_at_exit(void);
 // thread local pointer to stats node within list
 extern __thread kmp_stats_list* __kmp_stats_thread_ptr;
 // head to stats list.
-extern kmp_stats_list __kmp_stats_list;
+extern kmp_stats_list* __kmp_stats_list;
 // lock for __kmp_stats_list
 extern kmp_tas_lock_t  __kmp_stats_lock;
 // reference start time
 extern tsc_tick_count __kmp_stats_start_time;
 // interface to output
 extern kmp_stats_output_module __kmp_stats_output;
 
 #ifdef __cplusplus

 #define KMP_RESET_STATS()  __kmp_reset_stats()
 
 #if (KMP_DEVELOPER_STATS)
 # define KMP_TIME_DEVELOPER_BLOCK(n)             KMP_TIME_BLOCK(n)
 # define KMP_COUNT_DEVELOPER_VALUE(n,v)          KMP_COUNT_VALUE(n,v)
 # define KMP_COUNT_DEVELOPER_BLOCK(n)            KMP_COUNT_BLOCK(n)
 # define KMP_START_DEVELOPER_EXPLICIT_TIMER(n)   KMP_START_EXPLICIT_TIMER(n)
 # define KMP_STOP_DEVELOPER_EXPLICIT_TIMER(n)    KMP_STOP_EXPLICIT_TIMER(n)
+# define KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(n) KMP_TIME_PARTITIONED_BLOCK(n)
 #else
 // Null definitions
 # define KMP_TIME_DEVELOPER_BLOCK(n)             ((void)0)
 # define KMP_COUNT_DEVELOPER_VALUE(n,v)          ((void)0)
 # define KMP_COUNT_DEVELOPER_BLOCK(n)            ((void)0)
 # define KMP_START_DEVELOPER_EXPLICIT_TIMER(n)   ((void)0)
 # define KMP_STOP_DEVELOPER_EXPLICIT_TIMER(n)    ((void)0)
+# define KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(n) ((void)0)
 #endif
 
 #else // KMP_STATS_ENABLED
 
 // Null definitions
 #define KMP_TIME_BLOCK(n)             ((void)0)
 #define KMP_COUNT_VALUE(n,v)          ((void)0)
 #define KMP_COUNT_BLOCK(n)            ((void)0)
 #define KMP_START_EXPLICIT_TIMER(n)   ((void)0)
 #define KMP_STOP_EXPLICIT_TIMER(n)    ((void)0)
 
 #define KMP_OUTPUT_STATS(heading_string) ((void)0)
 #define KMP_RESET_STATS()  ((void)0)
 
 #define KMP_TIME_DEVELOPER_BLOCK(n)             ((void)0)
 #define KMP_COUNT_DEVELOPER_VALUE(n,v)          ((void)0)
 #define KMP_COUNT_DEVELOPER_BLOCK(n)            ((void)0)
 #define KMP_START_DEVELOPER_EXPLICIT_TIMER(n)   ((void)0)
 #define KMP_STOP_DEVELOPER_EXPLICIT_TIMER(n)    ((void)0)
 #define KMP_INIT_PARTITIONED_TIMERS(name)       ((void)0)
 #define KMP_TIME_PARTITIONED_BLOCK(name)        ((void)0)
+#define KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(n) ((void)0)
 #define KMP_PUSH_PARTITIONED_TIMER(name)        ((void)0)
 #define KMP_POP_PARTITIONED_TIMER()             ((void)0)
 #define KMP_SET_THREAD_STATE(state_name)        ((void)0)
 #define KMP_GET_THREAD_STATE()                  ((void)0)
 #define KMP_SET_THREAD_STATE_BLOCK(state_name)  ((void)0)
 #endif  // KMP_STATS_ENABLED
 
 #endif // KMP_STATS_H

runtime/src/kmp_stats.cpp

 #include <ctime>
 
 #define STRINGIZE2(x) #x
 #define STRINGIZE(x) STRINGIZE2(x)
 
 #define expandName(name,flags,ignore)  {STRINGIZE(name),flags},
 statInfo timeStat::timerInfo[] = {
     KMP_FOREACH_TIMER(expandName,0)
-    {0,0}
+    {"TIMER_LAST", 0}
 };
 const statInfo counter::counterInfo[] = {
     KMP_FOREACH_COUNTER(expandName,0)
-    {0,0}
+    {"COUNTER_LAST", 0}
 };
 #undef expandName
 
 #define expandName(ignore1,ignore2,ignore3)  {0.0,0.0,0.0},
 kmp_stats_output_module::rgb_color kmp_stats_output_module::timerColorInfo[] = {
     KMP_FOREACH_TIMER(expandName,0)
     {0.0,0.0,0.0}
 };
const kmp_stats_output_module::rgb_color kmp_stats_output_module::globalColorArray[] = {
     {0.7, 0.6, 0.0}, // gold
     {0.0, 0.0, 0.0}  // black
 };
 
 // Ensure that the atexit handler only runs once.
 static uint32_t statsPrinted = 0;
 
 // output interface
-static kmp_stats_output_module __kmp_stats_global_output;
+static kmp_stats_output_module* __kmp_stats_global_output = NULL;
 
 /* ****************************************************** */
 /* ************* statistic member functions ************* */
 
 void statistic::addSample(double sample)
 {
     double delta = sample - meanVal;
 
void explicitTimer::start(timer_e timerEnumValue) {
     startTime = tsc_tick_count::now();
     totalPauseTime = 0;
     if(timeStat::logEvent(timerEnumValue)) {
         __kmp_stats_thread_ptr->incrementNestValue();
     }
     return;
 }
 
-void explicitTimer::stop(timer_e timerEnumValue) {
+void explicitTimer::stop(timer_e timerEnumValue, kmp_stats_list* stats_ptr /* = nullptr */) {
     if (startTime.getValue() == 0)
         return;
 
     tsc_tick_count finishTime = tsc_tick_count::now();
 
     //stat->addSample ((tsc_tick_count::now() - startTime).ticks());
     stat->addSample(((finishTime - startTime) - totalPauseTime).ticks());
 
     if(timeStat::logEvent(timerEnumValue)) {
-        __kmp_stats_thread_ptr->push_event(startTime.getValue() - __kmp_stats_start_time.getValue(), finishTime.getValue() - __kmp_stats_start_time.getValue(), __kmp_stats_thread_ptr->getNestValue(), timerEnumValue);
-        __kmp_stats_thread_ptr->decrementNestValue();
+        if(!stats_ptr)
+            stats_ptr = __kmp_stats_thread_ptr;
+        stats_ptr->push_event(startTime.getValue() - __kmp_stats_start_time.getValue(), finishTime.getValue() - __kmp_stats_start_time.getValue(), __kmp_stats_thread_ptr->getNestValue(), timerEnumValue);
+        stats_ptr->decrementNestValue();
     }
 
     /* We accept the risk that we drop a sample because it really did start at t==0. */
     startTime = 0;
     return;
 }
 
 /* ************************************************************** */

 }
 
 void kmp_stats_output_module::windupExplicitTimers()
 {
     // Wind up any explicit timers. We assume that it's fair at this point to just walk all the explcit timers in all threads
     // and say "it's over".
     // If the timer wasn't running, this won't record anything anyway.
     kmp_stats_list::iterator it;
-    for(it = __kmp_stats_list.begin(); it != __kmp_stats_list.end(); it++) {
+    for(it = __kmp_stats_list->begin(); it != __kmp_stats_list->end(); it++) {
         kmp_stats_list* ptr = *it;
         ptr->getPartitionedTimers()->windup();
         for (int timer=0; timer<EXPLICIT_TIMER_LAST; timer++) {
-            ptr->getExplicitTimer(explicit_timer_e(timer))->stop((timer_e)timer);
+            ptr->getExplicitTimer(explicit_timer_e(timer))->stop((timer_e)timer, ptr);
         }
     }
 }
 
 void kmp_stats_output_module::printPloticusFile() {
     int i;
-    int size = __kmp_stats_list.size();
+    int size = __kmp_stats_list->size();
     FILE* plotOut = fopen(plotFileName, "w+");
 
     fprintf(plotOut, "#proc page\n"
                      "   pagesize: 15 10\n"
                      "   scale: 1.0\n\n");
 
     fprintf(plotOut, "#proc getdata\n"
                      "   file: %s\n\n",
void kmp_stats_output_module::outputStats(const char* heading)
     if (eventPrintingEnabled()) {
         eventsOut = fopen(eventsFileName, "w+");
     }
 
     printHeaderInfo (statsOut);
     fprintf(statsOut, "%s\n",heading);
     // Accumulate across threads.
     kmp_stats_list::iterator it;
-    for (it = __kmp_stats_list.begin(); it != __kmp_stats_list.end(); it++) {
+    for (it = __kmp_stats_list->begin(); it != __kmp_stats_list->end(); it++) {
         int t = (*it)->getGtid();
         // Output per thread stats if requested.
         if (printPerThreadFlag) {
             fprintf (statsOut, "Thread %d\n", t);
             printTimerStats (statsOut, (*it)->getTimers(), 0);
             printCounters   (statsOut, (*it)->getCounters());
             fprintf (statsOut,"\n");
         }

 /* *************  exported C functions ************** */
 
 // no name mangling for these functions, we want the c files to be able to get at these functions
 extern "C" {
 
 void __kmp_reset_stats()
 {
     kmp_stats_list::iterator it;
-    for(it = __kmp_stats_list.begin(); it != __kmp_stats_list.end(); it++) {
+    for(it = __kmp_stats_list->begin(); it != __kmp_stats_list->end(); it++) {
         timeStat * timers     = (*it)->getTimers();
         counter * counters    = (*it)->getCounters();
         explicitTimer * eTimers = (*it)->getExplicitTimers();
 
         for (int t = 0; t<TIMER_LAST; t++)
             timers[t].reset();
 
         for (int c = 0; c<COUNTER_LAST; c++)
             counters[c].reset();
 
         for (int t=0; t<EXPLICIT_TIMER_LAST; t++)
             eTimers[t].reset();
 
         // reset the event vector so all previous events are "erased"
         (*it)->resetEventVector();
     }
 }
 
 // This function will reset all stats and stop all threads' explicit timers if they haven't been stopped already.
 void __kmp_output_stats(const char * heading)
 {
-    __kmp_stats_global_output.outputStats(heading);
+    __kmp_stats_global_output->outputStats(heading);
     __kmp_reset_stats();
 }
 
 void __kmp_accumulate_stats_at_exit(void)
 {
     // Only do this once.
     if (KMP_XCHG_FIXED32(&statsPrinted, 1) != 0)
         return;
 
     __kmp_output_stats("Statistics on exit");
 }
 
 void __kmp_stats_init(void)
 {
+    __kmp_init_tas_lock( & __kmp_stats_lock );
+    __kmp_stats_start_time = tsc_tick_count::now();
+    __kmp_stats_global_output = new kmp_stats_output_module();
+    __kmp_stats_list = new kmp_stats_list();
+}
+
+void __kmp_stats_fini(void)
+{
+    __kmp_accumulate_stats_at_exit();
+    __kmp_stats_list->deallocate();
+    delete __kmp_stats_global_output;
+    delete __kmp_stats_list;
 }
 
 } // extern "C"
 

runtime/src/z_Linux_util.c

 
 #if KMP_STATS_ENABLED
     // sets up worker thread stats
     __kmp_acquire_tas_lock(&__kmp_stats_lock, gtid);
 
     // th->th.th_stats is used to transfer thread specific stats-pointer to __kmp_launch_worker
     // So when thread is created (goes into __kmp_launch_worker) it will
     // set it's __thread local pointer to th->th.th_stats
-    th->th.th_stats = __kmp_stats_list.push_back(gtid);
-    if(KMP_UBER_GTID(gtid)) {
-        __kmp_stats_start_time = tsc_tick_count::now();
-        __kmp_stats_thread_ptr = th->th.th_stats;
-        __kmp_stats_init();
-        KMP_START_EXPLICIT_TIMER(OMP_worker_thread_life);
-        KMP_SET_THREAD_STATE(SERIAL_REGION);
-        KMP_INIT_PARTITIONED_TIMERS(OMP_serial);
+    if(!KMP_UBER_GTID(gtid)) {
+        th->th.th_stats = __kmp_stats_list->push_back(gtid);
+    } else {
+        // For root threads, the __kmp_stats_thread_ptr is set in __kmp_register_root(), so
+        // set the th->th.th_stats field to it.
+        th->th.th_stats = __kmp_stats_thread_ptr;
     }
     __kmp_release_tas_lock(&__kmp_stats_lock, gtid);
 
 #endif // KMP_STATS_ENABLED
 
     if ( KMP_UBER_GTID(gtid) ) {
         KA_TRACE( 10, ("__kmp_create_worker: uber thread (%d)\n", gtid ) );
         th -> th.th_info.ds.ds_thread = pthread_self();

 }
 
 /* This routine puts the calling thread to sleep after setting the
  * sleep bit for the indicated flag variable to true.
  */
 template <class C>
 static inline void __kmp_suspend_template( int th_gtid, C *flag )
 {
-    KMP_TIME_DEVELOPER_BLOCK(USER_suspend);
+    KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_suspend);
     kmp_info_t *th = __kmp_threads[th_gtid];
     int status;
     typename C::flag_t old_spin;
 
     KF_TRACE( 30, ("__kmp_suspend_template: T#%d enter for flag = %p\n", th_gtid, flag->get() ) );
 
     __kmp_suspend_initialize_thread( th );
 

 
 /* This routine signals the thread specified by target_gtid to wake up
  * after setting the sleep bit indicated by the flag argument to FALSE.
  * The target thread must already have called __kmp_suspend_template()
  */
 template <class C>
 static inline void __kmp_resume_template( int target_gtid, C *flag )
 {
-    KMP_TIME_DEVELOPER_BLOCK(USER_resume);
+    KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume);
     kmp_info_t *th = __kmp_threads[target_gtid];
     int status;
 
 #ifdef KMP_DEBUG
     int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
 #endif
 
     KF_TRACE( 30, ( "__kmp_resume_template: T#%d wants to wakeup T#%d enter\n", gtid, target_gtid ) );

 void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) {
     __kmp_resume_template(target_gtid, flag);
 }
 
 #if KMP_USE_MONITOR
 void
 __kmp_resume_monitor()
 {
-    KMP_TIME_DEVELOPER_BLOCK(USER_resume);
+    KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume);
     int status;
 #ifdef KMP_DEBUG
     int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
     KF_TRACE( 30, ( "__kmp_resume_monitor: T#%d wants to wakeup T#%d enter\n",
                     gtid, KMP_GTID_MONITOR ) );
     KMP_DEBUG_ASSERT( gtid != KMP_GTID_MONITOR );
 #endif
     status = pthread_mutex_lock( &__kmp_wait_mx.m_mutex );