[OpenMP][Stats] Fix stats gathering for distribute and team clause

[jlpeyton]

Summary

The distribute clause needs an explicit push of a timer. The teams
clause needs a timer added and also, similarly to parallel, exchanged
with the serial timer when encountered so that serial regions are counted properly.

[tlwilmar]

LGTM

openmp/trunk/runtime/src/kmp_csupport.cpp

 */
 void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,
                        ...) {
   int gtid = __kmp_entry_gtid();
   kmp_info_t *this_thr = __kmp_threads[gtid];
   va_list ap;
   va_start(ap, microtask);
 
+#if KMP_STATS_ENABLED
   KMP_COUNT_BLOCK(OMP_TEAMS);
+  stats_state_e previous_state = KMP_GET_THREAD_STATE();
+  if (previous_state == stats_state_e::SERIAL_REGION) {
+    KMP_EXCHANGE_PARTITIONED_TIMER(OMP_teams_overhead);
+  } else {
+    KMP_PUSH_PARTITIONED_TIMER(OMP_teams_overhead);
+  }
+#endif
 
   // remember teams entry point and nesting level
   this_thr->th.th_teams_microtask = microtask;
   this_thr->th.th_teams_level =
       this_thr->th.th_team->t.t_level; // AC: can be >0 on host
 
 #if OMPT_SUPPORT
   kmp_team_t *parent_team = this_thr->th.th_team;
#endif
   KMP_DEBUG_ASSERT(this_thr->th.th_cg_roots);
   this_thr->th.th_current_task->td_icvs.thread_limit =
       this_thr->th.th_cg_roots->cg_thread_limit;
 
   this_thr->th.th_teams_microtask = NULL;
   this_thr->th.th_teams_level = 0;
   *(kmp_int64 *)(&this_thr->th.th_teams_size) = 0L;
   va_end(ap);
+#if KMP_STATS_ENABLED
+  if (previous_state == stats_state_e::SERIAL_REGION) {
+    KMP_EXCHANGE_PARTITIONED_TIMER(OMP_serial);
+  } else {
+    KMP_POP_PARTITIONED_TIMER();
+  }
+#endif // KMP_STATS_ENABLED
 }
 #endif /* OMP_40_ENABLED */
 
 // I don't think this function should ever have been exported.
 // The __kmpc_ prefix was misapplied.  I'm fairly certain that no generated
 // openmp code ever called it, but it's been exported from the RTL for so
 // long that I'm afraid to remove the definition.
 int __kmpc_invoke_task_func(int gtid) { return __kmp_invoke_task_func(gtid); }

openmp/trunk/runtime/src/kmp_dispatch.cpp

if (ub >= lb) {
       // AC: cast to unsigned is needed for loops like (i=-2B; i<2B; i+=1B),
       // where the division needs to be unsigned regardless of the result type
       tc = (UT)(ub - lb) / st + 1;
     } else { // ub < lb
       tc = 0; // zero-trip
     }
   }
 
+#if KMP_STATS_ENABLED
+  if (KMP_MASTER_GTID(gtid)) {
+    KMP_COUNT_VALUE(OMP_loop_dynamic_total_iterations, tc);
+  }
+#endif
+
   pr->u.p.lb = lb;
   pr->u.p.ub = ub;
   pr->u.p.st = st;
   pr->u.p.tc = tc;
 
 #if KMP_OS_WINDOWS
   pr->u.p.last_upper = ub + st;
 #endif /* KMP_OS_WINDOWS */

openmp/trunk/runtime/src/kmp_runtime.cpp

if (call_context == fork_context_gnu) {
       return TRUE;
     }
 
     /* Invoke microtask for MASTER thread */
     KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid,
                   team->t.t_id, team->t.t_pkfn));
   } // END of timer KMP_fork_call block
 
+#if KMP_STATS_ENABLED && OMP_40_ENABLED
+  // If beginning a teams construct, then change thread state
+  stats_state_e previous_state = KMP_GET_THREAD_STATE();
+  if (!ap) {
+    KMP_SET_THREAD_STATE(stats_state_e::TEAMS_REGION);
+  }
+#endif
+
   if (!team->t.t_invoke(gtid)) {
     KMP_ASSERT2(0, "cannot invoke microtask for MASTER thread");
   }
+
+#if KMP_STATS_ENABLED && OMP_40_ENABLED
+  // If was beginning of a teams construct, then reset thread state
+  if (!ap) {
+    KMP_SET_THREAD_STATE(previous_state);
+  }
+#endif
+
   KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid,
                 team->t.t_id, team->t.t_pkfn));
   KMP_MB(); /* Flush all pending memory write invalidates.  */
 
   KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid));
 
 #if OMPT_SUPPORT
   if (ompt_enabled.enabled) {
if (ompt_enabled.ompt_callback_implicit_task) {
     ompt_team_size = team->t.t_nproc;
     ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
         ompt_scope_begin, my_parallel_data, my_task_data, ompt_team_size,
         __kmp_tid_from_gtid(gtid), ompt_task_implicit); // TODO: Can this be ompt_task_initial?
     OMPT_CUR_TASK_INFO(this_thr)->thread_num = __kmp_tid_from_gtid(gtid);
   }
 #endif
 
-  {
-    KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
-    KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
-    rc =
-        __kmp_invoke_microtask((microtask_t)TCR_SYNC_PTR(team->t.t_pkfn), gtid,
-                               tid, (int)team->t.t_argc, (void **)team->t.t_argv
+#if KMP_STATS_ENABLED
+  stats_state_e previous_state = KMP_GET_THREAD_STATE();
+  if (previous_state == stats_state_e::TEAMS_REGION) {
+    KMP_PUSH_PARTITIONED_TIMER(OMP_teams);
+  } else {
+    KMP_PUSH_PARTITIONED_TIMER(OMP_parallel);
+  }
+  KMP_SET_THREAD_STATE(IMPLICIT_TASK);
+#endif
+
+  rc = __kmp_invoke_microtask((microtask_t)TCR_SYNC_PTR(team->t.t_pkfn), gtid,
 #if OMPT_SUPPORT
-                               ,
-                               exit_runtime_p
 #endif
-                               );
 #if OMPT_SUPPORT
-    *exit_runtime_p = NULL;
 #endif
+
+#if KMP_STATS_ENABLED
+  if (previous_state == stats_state_e::TEAMS_REGION) {
+    KMP_SET_THREAD_STATE(previous_state);
   }
+  KMP_POP_PARTITIONED_TIMER();
+#endif
 
 #if USE_ITT_BUILD
   if (__itt_stack_caller_create_ptr) {
     __kmp_itt_stack_callee_leave(
         (__itt_caller)
             team->t.t_stack_id); // inform ittnotify about leaving user's code
   }
 #endif /* USE_ITT_BUILD */

openmp/trunk/runtime/src/kmp_sched.cpp

 char const *traits_t<int>::spec = "d";
 char const *traits_t<unsigned int>::spec = "u";
 char const *traits_t<long long>::spec = "lld";
 char const *traits_t<unsigned long long>::spec = "llu";
 char const *traits_t<long>::spec = "ld";
 //-------------------------------------------------------------------------
 #endif
 
+#if KMP_STATS_ENABLED
+#define KMP_STATS_LOOP_END(stat)                                               \
+  {                                                                            \
+    kmp_int64 t;                                                               \
+    kmp_int64 u = (kmp_int64)(*pupper);                                        \
+    kmp_int64 l = (kmp_int64)(*plower);                                        \
+    kmp_int64 i = (kmp_int64)incr;                                             \
+    if (i == 1) {                                                              \
+      t = u - l + 1;                                                           \
+    } else if (i == -1) {                                                      \
+      t = l - u + 1;                                                           \
+    } else if (i > 0) {                                                        \
+      t = (u - l) / i + 1;                                                     \
+    } else {                                                                   \
+      t = (l - u) / (-i) + 1;                                                  \
+    }                                                                          \
+    KMP_COUNT_VALUE(stat, t);                                                  \
+    KMP_POP_PARTITIONED_TIMER();                                               \
+  }
+#else
+#define KMP_STATS_LOOP_END(stat) /* Nothing */
+#endif
+
 template <typename T>
 static void __kmp_for_static_init(ident_t *loc, kmp_int32 global_tid,
                                   kmp_int32 schedtype, kmp_int32 *plastiter,
                                   T *plower, T *pupper,
                                   typename traits_t<T>::signed_t *pstride,
                                   typename traits_t<T>::signed_t incr,
                                   typename traits_t<T>::signed_t chunk
 #if OMPT_SUPPORT && OMPT_OPTIONAL

 
 #if OMPT_SUPPORT && OMPT_OPTIONAL
     if (ompt_enabled.ompt_callback_work) {
       ompt_callbacks.ompt_callback(ompt_callback_work)(
           ompt_work_type, ompt_scope_begin, &(team_info->parallel_data),
           &(task_info->task_data), 0, codeptr);
     }
 #endif
+    KMP_STATS_LOOP_END(OMP_loop_static_iterations);
     return;
   }
 
 #if OMP_40_ENABLED
   // Although there are schedule enumerations above kmp_ord_upper which are not
   // schedules for "distribute", the only ones which are useful are dynamic, so
   // cannot be seen here, since this codepath is only executed for static
   // schedules.

 
 #if OMPT_SUPPORT && OMPT_OPTIONAL
     if (ompt_enabled.ompt_callback_work) {
       ompt_callbacks.ompt_callback(ompt_callback_work)(
           ompt_work_type, ompt_scope_begin, &(team_info->parallel_data),
           &(task_info->task_data), *pstride, codeptr);
     }
 #endif
+    KMP_STATS_LOOP_END(OMP_loop_static_iterations);
     return;
   }
   nth = team->t.t_nproc;
   if (nth == 1) {
     if (plastiter != NULL)
       *plastiter = TRUE;
     *pstride =
         (incr > 0) ? (*pupper - *plower + 1) : (-(*plower - *pupper + 1));

 
 #if OMPT_SUPPORT && OMPT_OPTIONAL
     if (ompt_enabled.ompt_callback_work) {
       ompt_callbacks.ompt_callback(ompt_callback_work)(
           ompt_work_type, ompt_scope_begin, &(team_info->parallel_data),
           &(task_info->task_data), *pstride, codeptr);
     }
 #endif
+    KMP_STATS_LOOP_END(OMP_loop_static_iterations);
     return;
   }
 
   /* compute trip count */
   if (incr == 1) {
     trip_count = *pupper - *plower + 1;
   } else if (incr == -1) {
     trip_count = *plower - *pupper + 1;
   } else if (incr > 0) {
     // upper-lower can exceed the limit of signed type
     trip_count = (UT)(*pupper - *plower) / incr + 1;
   } else {
     trip_count = (UT)(*plower - *pupper) / (-incr) + 1;
   }
 
+#if KMP_STATS_ENABLED
+  if (KMP_MASTER_GTID(gtid)) {
+    KMP_COUNT_VALUE(OMP_loop_static_total_iterations, trip_count);
+  }
+#endif
+
   if (__kmp_env_consistency_check) {
     /* tripcount overflow? */
     if (trip_count == 0 && *pupper != *plower) {
       __kmp_error_construct(kmp_i18n_msg_CnsIterationRangeTooLarge, ct_pdo,
                             loc);
     }
   }
 

 #if OMPT_SUPPORT && OMPT_OPTIONAL
   if (ompt_enabled.ompt_callback_work) {
     ompt_callbacks.ompt_callback(ompt_callback_work)(
         ompt_work_type, ompt_scope_begin, &(team_info->parallel_data),
         &(task_info->task_data), trip_count, codeptr);
   }
 #endif
 
-#if KMP_STATS_ENABLED
-  {
-    kmp_int64 t;
-    kmp_int64 u = (kmp_int64)(*pupper);
-    kmp_int64 l = (kmp_int64)(*plower);
-    kmp_int64 i = (kmp_int64)incr;
-    /* compute trip count */
-    if (i == 1) {
-      t = u - l + 1;
-    } else if (i == -1) {
-      t = l - u + 1;
-    } else if (i > 0) {
-      t = (u - l) / i + 1;
-    } else {
-      t = (l - u) / (-i) + 1;
-    }
-    KMP_COUNT_VALUE(OMP_loop_static_iterations, t);
-    KMP_POP_PARTITIONED_TIMER();
-  }
-#endif
+  KMP_STATS_LOOP_END(OMP_loop_static_iterations);
   return;
 }
 
 template <typename T>
 static void __kmp_dist_for_static_init(ident_t *loc, kmp_int32 gtid,
                                        kmp_int32 schedule, kmp_int32 *plastiter,
                                        T *plower, T *pupper, T *pupperDist,
                                        typename traits_t<T>::signed_t *pstride,
                                        typename traits_t<T>::signed_t incr,
                                        typename traits_t<T>::signed_t chunk) {
   KMP_COUNT_BLOCK(OMP_DISTRIBUTE);
+  KMP_PUSH_PARTITIONED_TIMER(OMP_distribute);
+  KMP_PUSH_PARTITIONED_TIMER(OMP_distribute_scheduling);
   typedef typename traits_t<T>::unsigned_t UT;
   typedef typename traits_t<T>::signed_t ST;
   kmp_uint32 tid;
   kmp_uint32 nth;
   kmp_uint32 team_id;
   kmp_uint32 nteams;
   UT trip_count;
   kmp_team_t *team;
buff = __kmp_str_format(
         "stride=%%%s signed?<%s>\n",
         traits_t<T>::spec, traits_t<T>::spec, traits_t<T>::spec,
         traits_t<ST>::spec, traits_t<T>::spec);
     KD_TRACE(100, (buff, *plastiter, *plower, *pupper, *pupperDist, *pstride));
     __kmp_str_free(&buff);
   }
 #endif
   KE_TRACE(10, ("__kmpc_dist_for_static_init: T#%d return\n", gtid));
+  KMP_STATS_LOOP_END(OMP_distribute_iterations);
   return;
 }
 
 template <typename T>
 static void __kmp_team_static_init(ident_t *loc, kmp_int32 gtid,
                                    kmp_int32 *p_last, T *p_lb, T *p_ub,
                                    typename traits_t<T>::signed_t *p_st,
                                    typename traits_t<T>::signed_t incr,

openmp/trunk/runtime/src/kmp_stats.h

enum stats_state_e {
   IDLE,
   SERIAL_REGION,
   FORK_JOIN_BARRIER,
   PLAIN_BARRIER,
   TASKWAIT,
   TASKYIELD,
   TASKGROUP,
   IMPLICIT_TASK,
-  EXPLICIT_TASK
+  EXPLICIT_TASK,
+  TEAMS_REGION
 };
 
 /*!
  * \brief Add new counters under KMP_FOREACH_COUNTER() macro in kmp_stats.h
  *
  * @param macro a user defined macro that takes three arguments -
  * macro(COUNTER_NAME, flags, arg)
  * @param arg a user defined argument to send to the user defined macro

  *
  * @ingroup STATS_GATHERING2
  */
 // clang-format off
 #define KMP_FOREACH_TIMER(macro, arg)                                          \
   macro (OMP_worker_thread_life, stats_flags_e::logEvent, arg)                 \
   macro (OMP_parallel, stats_flags_e::logEvent, arg)                           \
   macro (OMP_parallel_overhead, stats_flags_e::logEvent, arg)                  \
+  macro (OMP_teams, stats_flags_e::logEvent, arg)                              \
+  macro (OMP_teams_overhead, stats_flags_e::logEvent, arg)                     \
   macro (OMP_loop_static, 0, arg)                                              \
   macro (OMP_loop_static_scheduling, 0, arg)                                   \
   macro (OMP_loop_dynamic, 0, arg)                                             \
   macro (OMP_loop_dynamic_scheduling, 0, arg)                                  \
+  macro (OMP_distribute, 0, arg)                                               \
+  macro (OMP_distribute_scheduling, 0, arg)                                    \
   macro (OMP_critical, 0, arg)                                                 \
   macro (OMP_critical_wait, 0, arg)                                            \
   macro (OMP_single, 0, arg)                                                   \
   macro (OMP_master, 0, arg)                                                   \
   macro (OMP_task_immediate, 0, arg)                                           \
   macro (OMP_task_taskwait, 0, arg)                                            \
   macro (OMP_task_taskyield, 0, arg)                                           \
   macro (OMP_task_taskgroup, 0, arg)                                           \
   macro (OMP_task_join_bar, 0, arg)                                            \
   macro (OMP_task_plain_bar, 0, arg)                                           \
   macro (OMP_taskloop_scheduling, 0, arg)                                      \
   macro (OMP_plain_barrier, stats_flags_e::logEvent, arg)                      \
   macro (OMP_idle, stats_flags_e::logEvent, arg)                               \
   macro (OMP_fork_barrier, stats_flags_e::logEvent, arg)                       \
   macro (OMP_join_barrier, stats_flags_e::logEvent, arg)                       \
   macro (OMP_serial, stats_flags_e::logEvent, arg)                             \
   macro (OMP_set_numthreads, stats_flags_e::noUnits | stats_flags_e::noTotal,  \
          arg)                                                                  \
   macro (OMP_PARALLEL_args, stats_flags_e::noUnits | stats_flags_e::noTotal,   \
          arg)                                                                  \
   macro (OMP_loop_static_iterations,                                           \
          stats_flags_e::noUnits | stats_flags_e::noTotal, arg)                 \
+  macro (OMP_loop_static_total_iterations,                                     \
+         stats_flags_e::noUnits | stats_flags_e::noTotal, arg)                 \
   macro (OMP_loop_dynamic_iterations,                                          \
          stats_flags_e::noUnits | stats_flags_e::noTotal, arg)                 \
+  macro (OMP_loop_dynamic_total_iterations,                                    \
+         stats_flags_e::noUnits | stats_flags_e::noTotal, arg)                 \
+  macro (OMP_distribute_iterations,                                            \
+         stats_flags_e::noUnits | stats_flags_e::noTotal, arg)                 \
   KMP_FOREACH_DEVELOPER_TIMER(macro, arg)
 // clang-format on
 
 // OMP_worker_thread_life -- Time from thread becoming an OpenMP thread (either
 //                           initializing OpenMP or being created by a master)
 //                           until the thread is destroyed
 // OMP_parallel           -- Time thread spends executing work directly
 //                           within a #pragma omp parallel

openmp/trunk/runtime/src/kmp_stats.cpp

static std::string generateFilename(char const *prototype,
   }
   return res;
 }
 
 // init() is called very near the beginning of execution time in the constructor
 // of __kmp_stats_global_output
 void kmp_stats_output_module::init() {
 
-  fprintf(stderr, "*** Stats enabled OpenMP* runtime ***\n");
   char *statsFileName = getenv("KMP_STATS_FILE");
   eventsFileName = getenv("KMP_STATS_EVENTS_FILE");
   plotFileName = getenv("KMP_STATS_PLOT_FILE");
   char *threadStats = getenv("KMP_STATS_THREADS");
   char *threadEvents = getenv("KMP_STATS_EVENTS");
 
   // set the stats output filenames based on environment variables and defaults
   if (statsFileName) {