Fix OMPT state maintenance for barriers in OpenMP runtime. Fix missing initialization of implicit task id.

[jmellorcrummey]

Summary

Fix OMPT support for barriers so that state changes occur even if OMPT_TRACE turned off. These state changes are needed by performance tools that use callbacks for either ompt_event_wait_barrier_begin or ompt_event_wait_barrier_end.

Change ifdef flag to OMPT_BLAME for callbacks ompt_event_wait_barrier_begin or ompt_event_wait_barrier_end rather than OMPT_TRACE -- they were misclassified.

Without this patch, when the runtime is compiled with LIBOMP_OMPT_SUPPORT=true, LIBOMP_OMPT_BLAME=true, and LIBOMP_OMPT_TRACE=false, and a callback is registered for either ompt_event_wait_barrier_begin or ompt_event_wait_barrier_end,
then an assertion will trip.

Fix the scoping of one OMPT_TRACE ifdef, which should not have surrounded an update of an OMPT state.

Add a missing initialization of an OMPT task id for an implicit task.

[jmellorcrummey]

I am adding a comment to trigger reposting of this patch. I forgot to join openmp-commits before submitting this patch.

[jmellorcrummey]

Patch to correct handling of OMPT states and implicit task ids. I'm a newbie committer to LLVM, so any comments about how to accelerate getting this commit accepted would be appreciated.

runtime/src/kmp_barrier.cpp

 #if OMPT_SUPPORT
     ompt_task_id_t my_task_id;
     ompt_parallel_id_t my_parallel_id;
 #endif
 
     KA_TRACE(15, ("__kmp_barrier: T#%d(%d:%d) has arrived\n",
                   gtid, __kmp_team_from_gtid(gtid)->t.t_id, __kmp_tid_from_gtid(gtid)));
 
-#if OMPT_SUPPORT && OMPT_TRACE
+#if OMPT_SUPPORT
     if (ompt_status & ompt_status_track) {
+#if OMPT_BLAME
         if (ompt_status == ompt_status_track_callback) {
             my_task_id = team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id;
             my_parallel_id = team->t.ompt_team_info.parallel_id;
 
+#if OMPT_TRACE
             if (this_thr->th.ompt_thread_info.state == ompt_state_wait_single) {
                 if (ompt_callbacks.ompt_callback(ompt_event_single_others_end)) {
                     ompt_callbacks.ompt_callback(ompt_event_single_others_end)(
                         my_parallel_id, my_task_id);
                 }
             }
-            this_thr->th.ompt_thread_info.state = ompt_state_wait_barrier;
+#endif
             if (ompt_callbacks.ompt_callback(ompt_event_barrier_begin)) {
                 ompt_callbacks.ompt_callback(ompt_event_barrier_begin)(
                     my_parallel_id, my_task_id);
             }
-        } else {
-            this_thr->th.ompt_thread_info.state = ompt_state_wait_barrier;
-        }
+        } 
+#endif
+        // It is OK to report the barrier state after the barrier begin callback.
+        // According to the OMPT specification, a compliant implementation may
+        // even delay reporting this state until the barrier begins to wait.
+        this_thr->th.ompt_thread_info.state = ompt_state_wait_barrier;
     }
 #endif
 
     if (! team->t.t_serialized) {
 #if USE_ITT_BUILD
         // This value will be used in itt notify events below.
         void *itt_sync_obj = NULL;
 # if USE_ITT_NOTIFY

 #endif
         }
     }
     KA_TRACE(15, ("__kmp_barrier: T#%d(%d:%d) is leaving with return value %d\n",
                   gtid, __kmp_team_from_gtid(gtid)->t.t_id, __kmp_tid_from_gtid(gtid), status));
 
 #if OMPT_SUPPORT
     if (ompt_status & ompt_status_track) {
-#if OMPT_TRACE
+#if OMPT_BLAME
         if ((ompt_status == ompt_status_track_callback) &&
             ompt_callbacks.ompt_callback(ompt_event_barrier_end)) {
             ompt_callbacks.ompt_callback(ompt_event_barrier_end)(
                 my_parallel_id, my_task_id);
         }
 #endif
         this_thr->th.ompt_thread_info.state = ompt_state_work_parallel;
     }
#endif /* KMP_DEBUG */
 
     // Verify state
     KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]);
     KMP_DEBUG_ASSERT(TCR_PTR(this_thr->th.th_team));
     KMP_DEBUG_ASSERT(TCR_PTR(this_thr->th.th_root));
     KMP_DEBUG_ASSERT(this_thr == team->t.t_threads[tid]);
     KA_TRACE(10, ("__kmp_join_barrier: T#%d(%d:%d) arrived at join barrier\n", gtid, team_id, tid));
 
-#if OMPT_SUPPORT && OMPT_TRACE
+#if OMPT_SUPPORT 
+#if OMPT_TRACE
     if ((ompt_status == ompt_status_track_callback) &&
         ompt_callbacks.ompt_callback(ompt_event_barrier_begin)) {
         ompt_callbacks.ompt_callback(ompt_event_barrier_begin)(
             team->t.ompt_team_info.parallel_id,
             team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id);
     }
+#endif
     this_thr->th.ompt_thread_info.state = ompt_state_wait_barrier;
 #endif
 
     if (__kmp_tasking_mode == tskm_extra_barrier) {
         __kmp_tasking_barrier(team, this_thr, gtid);
         KA_TRACE(10, ("__kmp_join_barrier: T#%d(%d:%d) past taking barrier\n", gtid, team_id, tid));
     }
 # ifdef KMP_DEBUG

runtime/src/kmp_runtime.c

#endif
                 KA_TRACE(20, ("__kmp_launch_thread: T#%d(%d:%d) invoke microtask = %p\n",
                               gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid), (*pteam)->t.t_pkfn));
 
                 updateHWFPControl (*pteam);
 
 #if OMPT_SUPPORT
                 if (ompt_status & ompt_status_track) {
                     this_thr->th.ompt_thread_info.state = ompt_state_work_parallel;
+                    // Initialize OMPT task id for implicit task.
+                    int tid = __kmp_tid_from_gtid(gtid);
+                    (*pteam)->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id = 
+          __ompt_task_id_new(tid);
                 }
 #endif
 
                 KMP_STOP_EXPLICIT_TIMER(USER_launch_thread_loop);
                 {
                     KMP_TIME_BLOCK(USER_worker_invoke);
                     rc = (*pteam)->t.t_invoke( gtid );
                 }