Fix incorrect barrier imbalance reporting in ITTNOTIFY

[jlpeyton]

Summary

Exclude nested explicit tasks from timing, only outer level explicit task counted and its time added to barrier arrive time for the thread.

[hbae]

LGTM.

openmp/trunk/runtime/src/kmp_barrier.cpp

#if USE_ITT_BUILD
     if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
       __kmp_itt_barrier_finished(gtid, itt_sync_obj);
 #endif /* USE_ITT_BUILD */
   } else { // Team is serialized.
     status = 0;
     if (__kmp_tasking_mode != tskm_immediate_exec) {
 #if OMP_45_ENABLED
       if (this_thr->th.th_task_team != NULL) {
-        void *itt_sync_obj = NULL;
 #if USE_ITT_NOTIFY
+        void *itt_sync_obj = NULL;
         if (__itt_sync_create_ptr || KMP_ITT_DEBUG) {
           itt_sync_obj = __kmp_itt_barrier_object(gtid, bt, 1);
           __kmp_itt_barrier_starting(gtid, itt_sync_obj);
         }
 #endif
 
         KMP_DEBUG_ASSERT(this_thr->th.th_task_team->tt.tt_found_proxy_tasks ==
                          TRUE);

openmp/trunk/runtime/src/kmp_tasking.cpp

 //  __kmp_invoke_task: invoke the specified task
 //
 // gtid: global thread ID of caller
 // task: the task to invoke
 // current_task: the task to resume after task invokation
 static void __kmp_invoke_task(kmp_int32 gtid, kmp_task_t *task,
                               kmp_taskdata_t *current_task) {
   kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
+  kmp_info_t *thread;
 #if OMP_40_ENABLED
   int discard = 0 /* false */;
 #endif
   KA_TRACE(
       30, ("__kmp_invoke_task(enter): T#%d invoking task %p, current_task=%p\n",
            gtid, taskdata, current_task));
   KMP_DEBUG_ASSERT(task);
 #if OMP_45_ENABLED
if (taskdata->td_flags.proxy == TASK_PROXY &&
     KA_TRACE(30, ("__kmp_invoke_task(exit): T#%d completed bottom finish for "
                   "proxy task %p, resuming task %p\n",
                   gtid, taskdata, current_task));
 
     return;
   }
 #endif
 
-#if USE_ITT_BUILD && USE_ITT_NOTIFY
-  kmp_uint64 cur_time;
-  if (__kmp_forkjoin_frames_mode == 3) {
-    // Get the current time stamp to measure task execution time to correct
-    // barrier imbalance time
-    cur_time = __itt_get_timestamp();
-  }
-#endif
-
 #if OMPT_SUPPORT
   // For untied tasks, the first task executed only calls __kmpc_omp_task and
   // does not execute code.
   ompt_thread_info_t oldInfo;
-  kmp_info_t *thread;
   if (UNLIKELY(ompt_enabled.enabled)) {
     // Store the threads states and restore them after the task
     thread = __kmp_threads[gtid];
     oldInfo = thread->th.ompt_thread_info;
     thread->th.ompt_thread_info.wait_id = 0;
     thread->th.ompt_thread_info.state = (thread->th.th_team_serialized)
                                             ? omp_state_work_serial
                                             : omp_state_work_parallel;
#if OMP_45_ENABLED
   }
 #endif
 
 #if OMP_40_ENABLED
   // TODO: cancel tasks if the parallel region has also been cancelled
   // TODO: check if this sequence can be hoisted above __kmp_task_start
   // if cancellation has been enabled for this run ...
   if (__kmp_omp_cancellation) {
-    kmp_info_t *this_thr = __kmp_threads[gtid];
-    kmp_team_t *this_team = this_thr->th.th_team;
+    thread = __kmp_threads[gtid];
+    kmp_team_t *this_team = thread->th.th_team;
     kmp_taskgroup_t *taskgroup = taskdata->td_taskgroup;
     if ((taskgroup && taskgroup->cancel_request) ||
         (this_team->t.t_cancel_request == cancel_parallel)) {
 #if OMPT_SUPPORT && OMPT_OPTIONAL
       ompt_data_t *task_data;
       if (UNLIKELY(ompt_enabled.ompt_callback_cancel)) {
         __ompt_get_task_info_internal(0, NULL, &task_data, NULL, NULL, NULL);
         ompt_callbacks.ompt_callback(ompt_callback_cancel)(

 #endif // OMP_40_ENABLED
 
 // OMPT task begin
 #if OMPT_SUPPORT
     if (UNLIKELY(ompt_enabled.enabled))
       __ompt_task_start(task, current_task, gtid);
 #endif
 
+#if USE_ITT_BUILD && USE_ITT_NOTIFY
+    kmp_uint64 cur_time;
+    kmp_int32 kmp_itt_count_task =
+        __kmp_forkjoin_frames_mode == 3 && !taskdata->td_flags.task_serial &&
+        current_task->td_flags.tasktype == TASK_IMPLICIT;
+    if (kmp_itt_count_task) {
+      thread = __kmp_threads[gtid];
+      // Time outer level explicit task on barrier for adjusting imbalance time
+      if (thread->th.th_bar_arrive_time)
+        cur_time = __itt_get_timestamp();
+      else
+        kmp_itt_count_task = 0; // thread is not on a barrier - skip timing
+    }
+#endif
+
 #ifdef KMP_GOMP_COMPAT
     if (taskdata->td_flags.native) {
       ((void (*)(void *))(*(task->routine)))(task->shareds);
     } else
 #endif /* KMP_GOMP_COMPAT */
     {
       (*(task->routine))(gtid, task);
     }
     KMP_POP_PARTITIONED_TIMER();
 
+#if USE_ITT_BUILD && USE_ITT_NOTIFY
+    if (kmp_itt_count_task) {
+      // Barrier imbalance - adjust arrive time with the task duration
+      thread->th.th_bar_arrive_time += (__itt_get_timestamp() - cur_time);
+    }
+#endif
+
 #if OMP_40_ENABLED
   }
 #endif // OMP_40_ENABLED
 
 
 #if OMP_45_ENABLED
   // Proxy tasks are not handled by the runtime
   if (taskdata->td_flags.proxy != TASK_PROXY) {
 #endif
     ANNOTATE_HAPPENS_BEFORE(taskdata->td_parent);
 #if OMPT_SUPPORT
     if (UNLIKELY(ompt_enabled.enabled)) {
       thread->th.ompt_thread_info = oldInfo;
       if (taskdata->td_flags.tiedness == TASK_TIED) {
         taskdata->ompt_task_info.frame.exit_frame = NULL;
       }
       __kmp_task_finish<true>(gtid, task, current_task);
     } else
 #endif
       __kmp_task_finish<false>(gtid, task, current_task);
 #if OMP_45_ENABLED
   }
 #endif
 
-#if USE_ITT_BUILD && USE_ITT_NOTIFY
-  // Barrier imbalance - correct arrive time after the task finished
-  if (__kmp_forkjoin_frames_mode == 3) {
-    kmp_info_t *this_thr = __kmp_threads[gtid];
-    if (this_thr->th.th_bar_arrive_time) {
-      this_thr->th.th_bar_arrive_time += (__itt_get_timestamp() - cur_time);
-    }
-  }
-#endif
   KA_TRACE(
       30,
       ("__kmp_invoke_task(exit): T#%d completed task %p, resuming task %p\n",
        gtid, taskdata, current_task));
   return;
 }
 
 // __kmpc_omp_task_parts: Schedule a thread-switchable task for execution