ittnotify fixes

[jlpeyton]

Summary

Added time-taking into hierarchical barrier (to fix CQ562639).
Added the check of the nested level to all notifications, restructure all checks to be consistent.
Disabled notifications from the teams construct (because all the parallel regions in the teams are nested from library POV).

[jlpeyton]

*Ping*

[hfinkel]

This looks fine, but if the changes can be separated upon commit, that would be great.

[jlpeyton]

Closing the revision.

runtime/src/kmp_barrier.cpp

__kmp_linear_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
     register kmp_info_t **other_threads = team->t.t_threads;
 
     KA_TRACE(20, ("__kmp_linear_barrier_gather: T#%d(%d:%d) enter for barrier type %d\n",
                   gtid, team->t.t_id, tid, bt));
     KMP_DEBUG_ASSERT(this_thr == other_threads[this_thr->th.th_info.ds.ds_tid]);
 
 #if USE_ITT_BUILD && USE_ITT_NOTIFY
     // Barrier imbalance - save arrive time to the thread
-    if(__kmp_forkjoin_frames_mode == 2 || __kmp_forkjoin_frames_mode == 3) {
+    if(__kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 2) {
         this_thr->th.th_bar_arrive_time = this_thr->th.th_bar_min_time = __itt_get_timestamp();
     }
 #endif
     // We now perform a linear reduction to signal that all of the threads have arrived.
     if (!KMP_MASTER_TID(tid)) {
         KA_TRACE(20, ("__kmp_linear_barrier_gather: T#%d(%d:%d) releasing T#%d(%d:%d)"
                       "arrived(%p): %u => %u\n", gtid, team->t.t_id, tid,
                       __kmp_gtid_from_tid(0, team), team->t.t_id, 0, &thr_bar->b_arrived,
#endif /* KMP_CACHE_MANAGE */
                             &other_threads[i]->th.th_bar[bt].bb.b_arrived, new_state));
 
             // Wait for worker thread to arrive
             kmp_flag_64 flag(&other_threads[i]->th.th_bar[bt].bb.b_arrived, new_state);
             flag.wait(this_thr, FALSE
                       USE_ITT_BUILD_ARG(itt_sync_obj) );
 #if USE_ITT_BUILD && USE_ITT_NOTIFY
             // Barrier imbalance - write min of the thread time and the other thread time to the thread.
-            if (__kmp_forkjoin_frames_mode == 2 || __kmp_forkjoin_frames_mode == 3) {
+            if (__kmp_forkjoin_frames_mode == 2) {
                 this_thr->th.th_bar_min_time = KMP_MIN(this_thr->th.th_bar_min_time,
                                                           other_threads[i]->th.th_bar_min_time);
             }
 #endif
             if (reduce) {
                 KA_TRACE(100, ("__kmp_linear_barrier_gather: T#%d(%d:%d) += T#%d(%d:%d)\n", gtid,
                                team->t.t_id, tid, __kmp_gtid_from_tid(i, team), team->t.t_id, i));
                 (*reduce)(this_thr->th.th_local.reduce_data,
__kmp_tree_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
     register kmp_uint new_state;
 
     KA_TRACE(20, ("__kmp_tree_barrier_gather: T#%d(%d:%d) enter for barrier type %d\n",
                   gtid, team->t.t_id, tid, bt));
     KMP_DEBUG_ASSERT(this_thr == other_threads[this_thr->th.th_info.ds.ds_tid]);
 
 #if USE_ITT_BUILD && USE_ITT_NOTIFY
     // Barrier imbalance - save arrive time to the thread
-    if(__kmp_forkjoin_frames_mode == 2 || __kmp_forkjoin_frames_mode == 3) {
+    if(__kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 2) {
         this_thr->th.th_bar_arrive_time = this_thr->th.th_bar_min_time = __itt_get_timestamp();
     }
 #endif
     // Perform tree gather to wait until all threads have arrived; reduce any required data as we go
     child_tid = (tid << branch_bits) + 1;
     if (child_tid < nproc) {
         // Parent threads wait for all their children to arrive
         new_state = team->t.t_bar[bt].b_arrived + KMP_BARRIER_STATE_BUMP;
#endif /* KMP_CACHE_MANAGE */
                             __kmp_gtid_from_tid(child_tid, team), team->t.t_id, child_tid,
                             &child_bar->b_arrived, new_state));
             // Wait for child to arrive
             kmp_flag_64 flag(&child_bar->b_arrived, new_state);
             flag.wait(this_thr, FALSE
                       USE_ITT_BUILD_ARG(itt_sync_obj) );
 #if USE_ITT_BUILD && USE_ITT_NOTIFY
             // Barrier imbalance - write min of the thread time and a child time to the thread.
-            if (__kmp_forkjoin_frames_mode == 2 || __kmp_forkjoin_frames_mode == 3) {
+            if (__kmp_forkjoin_frames_mode == 2) {
                 this_thr->th.th_bar_min_time = KMP_MIN(this_thr->th.th_bar_min_time,
                                                           child_thr->th.th_bar_min_time);
             }
 #endif
             if (reduce) {
                 KA_TRACE(100, ("__kmp_tree_barrier_gather: T#%d(%d:%d) += T#%d(%d:%u)\n",
                                gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
                                team->t.t_id, child_tid));
__kmp_hyper_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
 
     KA_TRACE(20, ("__kmp_hyper_barrier_gather: T#%d(%d:%d) enter for barrier type %d\n",
                   gtid, team->t.t_id, tid, bt));
 
     KMP_DEBUG_ASSERT(this_thr == other_threads[this_thr->th.th_info.ds.ds_tid]);
 
 #if USE_ITT_BUILD && USE_ITT_NOTIFY
     // Barrier imbalance - save arrive time to the thread
-    if(__kmp_forkjoin_frames_mode == 2 || __kmp_forkjoin_frames_mode == 3) {
+    if(__kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 2) {
         this_thr->th.th_bar_arrive_time = this_thr->th.th_bar_min_time = __itt_get_timestamp();
     }
 #endif
     /* Perform a hypercube-embedded tree gather to wait until all of the threads have
        arrived, and reduce any required data as we go.  */
     kmp_flag_64 p_flag(&thr_bar->b_arrived);
     for (level=0, offset=1; offset<num_threads; level+=branch_bits, offset<<=branch_bits)
     {
#endif /* KMP_CACHE_MANAGE */
                           __kmp_gtid_from_tid(child_tid, team), team->t.t_id, child_tid,
                           &child_bar->b_arrived, new_state));
             // Wait for child to arrive
             kmp_flag_64 c_flag(&child_bar->b_arrived, new_state);
             c_flag.wait(this_thr, FALSE
                         USE_ITT_BUILD_ARG(itt_sync_obj) );
 #if USE_ITT_BUILD && USE_ITT_NOTIFY
             // Barrier imbalance - write min of the thread time and a child time to the thread.
-            if (__kmp_forkjoin_frames_mode == 2 || __kmp_forkjoin_frames_mode == 3) {
+            if (__kmp_forkjoin_frames_mode == 2) {
                 this_thr->th.th_bar_min_time = KMP_MIN(this_thr->th.th_bar_min_time,
                                                           child_thr->th.th_bar_min_time);
             }
 #endif
             if (reduce) {
                 KA_TRACE(100, ("__kmp_hyper_barrier_gather: T#%d(%d:%d) += T#%d(%d:%u)\n",
                                gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
                                team->t.t_id, child_tid));
if (other_threads[0]->th.th_teams_microtask)    // are we inside the teams construct?
             ++level; // level was not increased in teams construct for team_of_masters
     if (level == 1) thr_bar->use_oncore_barrier = 1;
     else thr_bar->use_oncore_barrier = 0; // Do not use oncore barrier when nested
 
     KA_TRACE(20, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) enter for barrier type %d\n",
                   gtid, team->t.t_id, tid, bt));
     KMP_DEBUG_ASSERT(this_thr == other_threads[this_thr->th.th_info.ds.ds_tid]);
 
+#if USE_ITT_BUILD && USE_ITT_NOTIFY
+    // Barrier imbalance - save arrive time to the thread
+    if(__kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 2) {
+        this_thr->th.th_bar_arrive_time = __itt_get_timestamp();
+    }
+#endif
+
     (void)__kmp_init_hierarchical_barrier_thread(bt, thr_bar, nproc, gtid, tid, team);
 
     if (thr_bar->my_level) { // not a leaf (my_level==0 means leaf)
         register kmp_int32 child_tid;
         new_state = (kmp_uint64)team->t.t_bar[bt].b_arrived + KMP_BARRIER_STATE_BUMP;
         if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME && thr_bar->use_oncore_barrier) {
             if (thr_bar->leaf_kids) { // First, wait for leaf children to check-in on my b_arrived flag
                 kmp_uint64 leaf_state = KMP_MASTER_TID(tid) ? thr_bar->b_arrived | thr_bar->leaf_state : (kmp_uint64)team->t.t_bar[bt].b_arrived | thr_bar->leaf_state;

 #if USE_ITT_BUILD
             /* TODO: In case of split reduction barrier, master thread may send acquired event early,
                before the final summation into the shared variable is done (final summation can be a
                long operation for array reductions).  */
             if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
                 __kmp_itt_barrier_middle(gtid, itt_sync_obj);
 #endif /* USE_ITT_BUILD */
 #if USE_ITT_BUILD && USE_ITT_NOTIFY
-            // Barrier - report frame end
-            if (__itt_frame_submit_v3_ptr && __kmp_forkjoin_frames_mode) {
+            // Barrier - report frame end (only if active_level == 1)
+            if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) && __kmp_forkjoin_frames_mode &&
+#if OMP_40_ENABLED
+                this_thr->th.th_teams_microtask == NULL &&
+#endif
+                team->t.t_active_level == 1)
+            {
                 kmp_uint64 cur_time = __itt_get_timestamp();
-                kmp_info_t **other_threads = this_thr->th.th_team->t.t_threads;
+                kmp_info_t **other_threads = team->t.t_threads;
                 int nproc = this_thr->th.th_team_nproc;
                 int i;
-                // Initialize with master's wait time
-                kmp_uint64 delta = cur_time - this_thr->th.th_bar_arrive_time;
                 switch(__kmp_forkjoin_frames_mode) {
                 case 1:
                     __kmp_itt_frame_submit(gtid, this_thr->th.th_frame_time, cur_time, 0, loc, nproc);
                     this_thr->th.th_frame_time = cur_time;
                     break;
-                case 2:
+                case 2: // AC 2015-01-19: currently does not work for hierarchical (to be fixed)
                     __kmp_itt_frame_submit(gtid, this_thr->th.th_bar_min_time, cur_time, 1, loc, nproc);
                     break;
                 case 3:
                     if( __itt_metadata_add_ptr ) {
+                        // Initialize with master's wait time
+                        kmp_uint64 delta = cur_time - this_thr->th.th_bar_arrive_time;
                         for (i=1; i<nproc; ++i) {
                             delta += ( cur_time - other_threads[i]->th.th_bar_arrive_time );
                         }
                         __kmp_itt_metadata_imbalance(gtid, this_thr->th.th_frame_time, cur_time, delta, (kmp_uint64)( reduce != NULL));
                     }
                     __kmp_itt_frame_submit(gtid, this_thr->th.th_frame_time, cur_time, 0, loc, nproc);
                     this_thr->th.th_frame_time = cur_time;
                     break;
if (KMP_MASTER_TID(tid)) {
         }
 #if USE_ITT_BUILD
         if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
             __kmp_itt_barrier_middle(gtid, itt_sync_obj);
 #endif /* USE_ITT_BUILD */
 
 # if USE_ITT_BUILD && USE_ITT_NOTIFY
         // Join barrier - report frame end
-        if (__itt_frame_submit_v3_ptr && __kmp_forkjoin_frames_mode) {
+        if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) && __kmp_forkjoin_frames_mode &&
+#if OMP_40_ENABLED
+            this_thr->th.th_teams_microtask == NULL &&
+#endif
+            team->t.t_active_level == 1)
+        {
             kmp_uint64 cur_time = __itt_get_timestamp();
             ident_t * loc = team->t.t_ident;
-            kmp_info_t **other_threads = this_thr->th.th_team->t.t_threads;
+            kmp_info_t **other_threads = team->t.t_threads;
             int nproc = this_thr->th.th_team_nproc;
             int i;
-            // Initialize with master's wait time
-            kmp_uint64 delta = cur_time - this_thr->th.th_bar_arrive_time;
             switch(__kmp_forkjoin_frames_mode) {
             case 1:
                 __kmp_itt_frame_submit(gtid, this_thr->th.th_frame_time, cur_time, 0, loc, nproc);
                 break;
             case 2:
                 __kmp_itt_frame_submit(gtid, this_thr->th.th_bar_min_time, cur_time, 1, loc, nproc);
                 break;
             case 3:
                 if( __itt_metadata_add_ptr ) {
+                    // Initialize with master's wait time
+                    kmp_uint64 delta = cur_time - this_thr->th.th_bar_arrive_time;
                     for (i=1; i<nproc; ++i) {
                         delta += ( cur_time - other_threads[i]->th.th_bar_arrive_time );
                     }
                     __kmp_itt_metadata_imbalance(gtid, this_thr->th.th_frame_time, cur_time, delta, 0);
                 }
                 __kmp_itt_frame_submit(gtid, this_thr->th.th_frame_time, cur_time, 0, loc, nproc);
                 this_thr->th.th_frame_time = cur_time;
                 break;

runtime/src/kmp_csupport.c

#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
             KA_TRACE( 20, ( "__kmpc_end_serialized_parallel: T#%d decreasing nesting depth of serial team %p to %d\n",
                             global_tid, serial_team, serial_team -> t.t_serialized ) );
         }
     }
 
 #if USE_ITT_BUILD
     kmp_uint64 cur_time = 0;
 #if  USE_ITT_NOTIFY
-    if( __itt_get_timestamp_ptr ) {
+    if ( __itt_get_timestamp_ptr ) {
         cur_time = __itt_get_timestamp();
     }
 #endif /* USE_ITT_NOTIFY */
+    if ( this_thr->th.th_team->t.t_level == 0
+#if OMP_40_ENABLED
+        && this_thr->th.th_teams_microtask == NULL
+#endif
+    ) {
         // Report the barrier
-    if( ( __kmp_forkjoin_frames_mode == 1 || __kmp_forkjoin_frames_mode == 3 ) && __itt_frame_submit_v3_ptr ) {
-        if( this_thr->th.th_team->t.t_level == 0 ) {
-            __kmp_itt_frame_submit( global_tid, this_thr->th.th_frame_time_serialized, cur_time, 0, loc, this_thr->th.th_team_nproc, 0 );
-        }
-    }
-    // Mark the end of the "parallel" region for VTune. Only use one of frame notification scheme at the moment.
-    if ( ( __itt_frame_end_v3_ptr && __kmp_forkjoin_frames && ! __kmp_forkjoin_frames_mode ) || KMP_ITT_DEBUG )
-    {
         this_thr->th.th_ident = loc;
-        __kmp_itt_region_joined( global_tid, 1 );
-    }
-    if ( ( __itt_frame_submit_v3_ptr && __kmp_forkjoin_frames_mode == 3 ) || KMP_ITT_DEBUG )
+        if ( ( __itt_frame_submit_v3_ptr || KMP_ITT_DEBUG ) &&
+            ( __kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 1 ) )
         {
-        this_thr->th.th_ident = loc;
+            __kmp_itt_frame_submit( global_tid, this_thr->th.th_frame_time_serialized,
+                                    cur_time, 0, loc, this_thr->th.th_team_nproc, 0 );
+            if ( __kmp_forkjoin_frames_mode == 3 )
                 // Since barrier frame for serialized region is equal to the region we use the same begin timestamp as for the barrier.
-        __kmp_itt_frame_submit( global_tid, serial_team->t.t_region_time, cur_time, 0, loc, this_thr->th.th_team_nproc, 2 );
+                __kmp_itt_frame_submit( global_tid, serial_team->t.t_region_time,
+                                        cur_time, 0, loc, this_thr->th.th_team_nproc, 2 );
+        } else if ( ( __itt_frame_end_v3_ptr || KMP_ITT_DEBUG ) &&
+            ! __kmp_forkjoin_frames_mode && __kmp_forkjoin_frames )
+            // Mark the end of the "parallel" region for VTune. Only use one of frame notification scheme at the moment.
+            __kmp_itt_region_joined( global_tid, 1 );
     }
 #endif /* USE_ITT_BUILD */
 
     if ( __kmp_env_consistency_check )
         __kmp_pop_parallel( global_tid, NULL );
 }
 
 /*!

runtime/src/kmp_dispatch.cpp

#endif
     /* setup data */
     th     = __kmp_threads[ gtid ];
     team   = th -> th.th_team;
     active = ! team -> t.t_serialized;
     th->th.th_ident = loc;
 
 #if USE_ITT_BUILD
     kmp_uint64 cur_chunk = chunk;
+    int itt_need_metadata_reporting = __itt_metadata_add_ptr && __kmp_forkjoin_frames_mode == 3 &&
+        KMP_MASTER_GTID(gtid) &&
+#if OMP_40_ENABLED
+        th->th.th_teams_microtask == NULL &&
+#endif
+        team->t.t_active_level == 1;
 #endif
     if ( ! active ) {
         pr = reinterpret_cast< dispatch_private_info_template< T >* >
             ( th -> th.th_dispatch -> th_disp_buffer ); /* top of the stack */
     } else {
         KMP_DEBUG_ASSERT( th->th.th_dispatch ==
                 &th->th.th_team->t.t_dispatch[th->th.th_info.ds.ds_tid] );
 
case kmp_sch_static_balanced:
                     // zero trip count
                     pr->u.p.count = 1;  /* means no more chunks to execute */
                     pr->u.p.parm1 = FALSE;
                     break;
                 }
             }
 #if USE_ITT_BUILD
             // Calculate chunk for metadata report
-            if(  __itt_metadata_add_ptr  && __kmp_forkjoin_frames_mode == 3 ) {
+            if ( itt_need_metadata_reporting )
                 cur_chunk = limit - init + 1;
-            }
 #endif
             if ( st == 1 ) {
                 pr->u.p.lb = lb + init;
                 pr->u.p.ub = lb + limit;
             } else {
                 T ub_tmp = lb + limit * st;   // calculated upper bound, "ub" is user-defined upper bound
                 pr->u.p.lb = lb + init * st;
                 // adjust upper bound to "ub" if needed, so that MS lastprivate will match it exactly
if ( active ) {
                         gtid, my_buffer_index, sh->buffer_index) );
 
         th -> th.th_dispatch -> th_dispatch_pr_current = (dispatch_private_info_t*) pr;
         th -> th.th_dispatch -> th_dispatch_sh_current = (dispatch_shared_info_t*)  sh;
 #if USE_ITT_BUILD
         if ( pr->ordered ) {
             __kmp_itt_ordered_init( gtid );
         }; // if
-#endif /* USE_ITT_BUILD */
-    }; // if
-
-#if USE_ITT_BUILD
         // Report loop metadata
-    if( __itt_metadata_add_ptr  && __kmp_forkjoin_frames_mode == 3 ) {
-        kmp_uint32 tid  = __kmp_tid_from_gtid( gtid );
-        if (KMP_MASTER_TID(tid)) {
+        if ( itt_need_metadata_reporting ) {
+            // Only report metadata by master of active team at level 1
             kmp_uint64 schedtype = 0;
-
             switch ( schedule ) {
             case kmp_sch_static_chunked:
             case kmp_sch_static_balanced:// Chunk is calculated in the switch above
                 break;
             case kmp_sch_static_greedy:
                 cur_chunk = pr->u.p.parm1;
                 break;
             case kmp_sch_dynamic_chunked:
                 schedtype = 1;
                 break;
             case kmp_sch_guided_iterative_chunked:
             case kmp_sch_guided_analytical_chunked:
                 schedtype = 2;
                 break;
             default:
 //            Should we put this case under "static"?
 //            case kmp_sch_static_steal:
                 schedtype = 3;
                 break;
             }
             __kmp_itt_metadata_loop(loc, schedtype, tc, cur_chunk);
         }
-    }
 #endif /* USE_ITT_BUILD */
+    }; // if
 
     #ifdef KMP_DEBUG
     {
         const char * buff;
         // create format specifiers before the debug output
         buff = __kmp_str_format(
             "__kmp_dispatch_init: T#%%d returning: schedule:%%d ordered:%%%s lb:%%%s ub:%%%s" \
             " st:%%%s tc:%%%s count:%%%s\n\tordered_lower:%%%s ordered_upper:%%%s" \

runtime/src/kmp_runtime.c

if ( team->t.t_serialized ) {
 
         ++th->th.th_local.this_construct;
         /* try to set team count to thread count--success means thread got the
            single block
         */
         /* TODO: Should this be acquire or release? */
         status = KMP_COMPARE_AND_STORE_ACQ32(&team->t.t_construct, old_this,
                                              th->th.th_local.this_construct);
+#if USE_ITT_BUILD
+        if ( __itt_metadata_add_ptr && __kmp_forkjoin_frames_mode == 3 && KMP_MASTER_GTID(gtid) &&
+#if OMP_40_ENABLED
+            th->th.th_teams_microtask == NULL &&
+#endif
+            team->t.t_active_level == 1 )
+        {   // Only report metadata by master of active team at level 1
+            __kmp_itt_metadata_single( id_ref );
+        }
+#endif /* USE_ITT_BUILD */
     }
 
     if( __kmp_env_consistency_check ) {
         if (status && push_ws) {
             __kmp_push_workshare( gtid, ct_psingle, id_ref );
         } else {
             __kmp_check_workshare( gtid, ct_psingle, id_ref );
         }
     }
 #if USE_ITT_BUILD
     if ( status ) {
         __kmp_itt_single_start( gtid );
     }
-    if( __itt_metadata_add_ptr && __kmp_forkjoin_frames_mode == 3 && KMP_MASTER_GTID(gtid)) {
-        __kmp_itt_metadata_single( id_ref );
-    }
-
 #endif /* USE_ITT_BUILD */
     return status;
 }
 
 void
 __kmp_exit_single( int gtid )
 {
 #if USE_ITT_BUILD
#endif /* OMP_40_ENABLED */
         KMP_MB();
     }
 
     if ( __kmp_env_consistency_check )
         __kmp_push_parallel( global_tid, NULL );
 
 #if USE_ITT_BUILD
     // Mark the start of the "parallel" region for VTune. Only use one of frame notification scheme at the moment
-    if ( ( __itt_frame_begin_v3_ptr && __kmp_forkjoin_frames && ! __kmp_forkjoin_frames_mode ) || KMP_ITT_DEBUG )
+    if ( serial_team->t.t_level == 1
+#if OMP_40_ENABLED
+        && this_thr->th.th_teams_microtask == NULL
+#endif
+    ) {
+#if USE_ITT_NOTIFY
+        // Save the start of the "parallel" region for VTune. This is the frame begin at the same time.
+        if ( ( __itt_get_timestamp_ptr || KMP_ITT_DEBUG ) &&
+            ( __kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 1 ) )
+        {
+             serial_team->t.t_region_time = this_thr->th.th_frame_time_serialized = __itt_get_timestamp();
+        } else // only one notification scheme (either "submit" or "forking/joined", not both)
+#endif
+        if ( ( __itt_frame_begin_v3_ptr || KMP_ITT_DEBUG ) &&
+             __kmp_forkjoin_frames && ! __kmp_forkjoin_frames_mode )
         {
             this_thr->th.th_ident = loc;
             // 0 - no barriers; 1 - serialized parallel
             __kmp_itt_region_forking( global_tid, this_thr->th.th_team_nproc, 0, 1 );
         }
-    // Save the start of the "parallel" region for VTune. This is the join barrier begin at the same time.
-    if( ( ( __kmp_forkjoin_frames_mode == 1 || __kmp_forkjoin_frames_mode == 3 ) &&
-          __itt_frame_submit_v3_ptr && __itt_get_timestamp_ptr ) || KMP_ITT_DEBUG )
-    {
-        this_thr->th.th_ident = loc;
-#if USE_ITT_NOTIFY
-        if( this_thr->th.th_team->t.t_level == 1 ) {
-            serial_team->t.t_region_time = this_thr->th.th_frame_time_serialized = __itt_get_timestamp();
-        }
-#endif
     }
 #endif /* USE_ITT_BUILD */
 }
 
 /* most of the work for a fork */
 /* return true if we really went parallel, false if serialized */
 int
 __kmp_fork_call(
#endif /* OMP_40_ENABLED */
     __kmp_fork_team_threads( root, team, master_th, gtid );
     __kmp_setup_icv_copy( team, nthreads, &master_th->th.th_current_task->td_icvs, loc );
 
 
     __kmp_release_bootstrap_lock( &__kmp_forkjoin_lock );
 
 
 #if USE_ITT_BUILD
-    // Mark start of "parallel" region for VTune. Only use one of frame notification scheme at the moment.
-    if ((__itt_frame_begin_v3_ptr && __kmp_forkjoin_frames && !__kmp_forkjoin_frames_mode) || KMP_ITT_DEBUG) {
+    if ( team->t.t_active_level == 1 // only report frames at level 1
 # if OMP_40_ENABLED
-        if (!master_th->th.th_teams_microtask || microtask == (microtask_t)__kmp_teams_master)
-            // Either not in teams or the outer fork of the teams construct
+        && !master_th->th.th_teams_microtask // not in teams construct
 # endif /* OMP_40_ENABLED */
-        {
-            __kmp_itt_region_forking(gtid, team->t.t_nproc, 0);
-        }
-    }
+    ) {
 #if USE_ITT_NOTIFY
+        if ( ( __itt_frame_submit_v3_ptr || KMP_ITT_DEBUG ) &&
+             ( __kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 1 ) )
+        {
             kmp_uint64 tmp_time = 0;
-    if (((__kmp_forkjoin_frames_mode == 1 || __kmp_forkjoin_frames_mode == 3) && __itt_frame_submit_v3_ptr) || KMP_ITT_DEBUG) {
-        if (!(team->t.t_active_level > 1)) {
-# if OMP_40_ENABLED
-            if (!master_th->th.th_teams_microtask || microtask == (microtask_t)__kmp_teams_master) {
-            // Either not in teams or the outer fork of the teams construct
-# endif /* OMP_40_ENABLED */
-                if ( __itt_get_timestamp_ptr )
-                    tmp_time = __itt_get_timestamp();
             // Internal fork - report frame begin
             master_th->th.th_frame_time  = tmp_time;
-                if ( __kmp_forkjoin_frames_mode==3 )
-                    team->t.t_region_time = tmp_time;
-# if OMP_40_ENABLED
-            }
-# endif /* OMP_40_ENABLED */
+            if ( __kmp_forkjoin_frames_mode == 3 )
+        } else // only one notification scheme (either "submit" or "forking/joined", not both)
+#endif /* USE_ITT_NOTIFY */
+        if ( ( __itt_frame_begin_v3_ptr || KMP_ITT_DEBUG ) &&
+             __kmp_forkjoin_frames && !__kmp_forkjoin_frames_mode )
+        { // Mark start of "parallel" region for VTune.
+            __kmp_itt_region_forking(gtid, team->t.t_nproc, 0);
         }
     }
-#endif /* USE_ITT_NOTIFY */
 #endif /* USE_ITT_BUILD */
 
     /* now go on and do the work */
     KMP_DEBUG_ASSERT( team == __kmp_threads[gtid]->th.th_team );
     KMP_MB();
     KF_TRACE(10, ("__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n",
                   root, team, master_th, gtid));
 
#endif /* OMP_40_ENABLED */
 
     KMP_MB();
 
 #if USE_ITT_BUILD
     if ( __itt_stack_caller_create_ptr ) {
         __kmp_itt_stack_caller_destroy( (__itt_caller)team->t.t_stack_id ); // destroy the stack stitching id after join barrier
     }
 
-    // Mark end of "parallel" region for VTune. Only use one of frame notification scheme at the moment.
-    if ( ( __itt_frame_end_v3_ptr && __kmp_forkjoin_frames && ! __kmp_forkjoin_frames_mode ) || KMP_ITT_DEBUG ) {
+    // Mark end of "parallel" region for VTune.
+    if ( team->t.t_active_level == 1
 # if OMP_40_ENABLED
-    if ( !master_th->th.th_teams_microtask /* not in teams */ ||
-         ( !exit_teams && team->t.t_level == master_th->th.th_teams_level ) )
-        // Either not in teams or exiting teams region
-        // (teams is a frame and no other frames inside the teams)
+        && !master_th->th.th_teams_microtask /* not in teams construct */
 # endif /* OMP_40_ENABLED */
-    {
+    ) {
         master_th->th.th_ident = loc;
+        // only one notification scheme (either "submit" or "forking/joined", not both)
+        if ( ( __itt_frame_submit_v3_ptr || KMP_ITT_DEBUG ) && __kmp_forkjoin_frames_mode == 3 )
+            __kmp_itt_frame_submit( gtid, team->t.t_region_time, master_th->th.th_frame_time,
+                                    0, loc, master_th->th.th_team_nproc, 1 );
+        else if ( ( __itt_frame_end_v3_ptr || KMP_ITT_DEBUG ) &&
+            ! __kmp_forkjoin_frames_mode && __kmp_forkjoin_frames )
             __kmp_itt_region_joined( gtid );
-    }
-    }
-    if ( ( __itt_frame_submit_v3_ptr && __kmp_forkjoin_frames_mode == 3 ) || KMP_ITT_DEBUG ) {
-# if OMP_40_ENABLED
-    if ( !master_th->th.th_teams_microtask /* not in teams */ ||
-         ( !exit_teams && team->t.t_level == master_th->th.th_teams_level ) )
-        // Either not in teams or exiting teams region
-        // (teams is a frame and no other frames inside the teams)
-# endif /* OMP_40_ENABLED */
-    {
-        master_th->th.th_ident = loc;
-        __kmp_itt_frame_submit( gtid, team->t.t_region_time, master_th->th.th_frame_time, 0, loc, master_th->th.th_team_nproc, 1 );
-    }
-    }
+    } // active_level == 1
 #endif /* USE_ITT_BUILD */
 
 #if OMP_40_ENABLED
     if ( master_th->th.th_teams_microtask &&
          !exit_teams &&
          team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
          team->t.t_level == master_th->th.th_teams_level + 1 ) {
         // AC: We need to leave the team structure intact at the end

runtime/src/kmp_sched.cpp

) {
     typedef typename traits_t< T >::unsigned_t  UT;
     typedef typename traits_t< T >::signed_t    ST;
     /*  this all has to be changed back to TID and such.. */
     register kmp_int32   gtid = global_tid;
     register kmp_uint32  tid;
     register kmp_uint32  nth;
     register UT          trip_count;
     register kmp_team_t *team;
+    register kmp_info_t *th = __kmp_threads[ gtid ];
 
     KMP_DEBUG_ASSERT( plastiter && plower && pupper && pstride );
     KE_TRACE( 10, ("__kmpc_for_static_init called (%d)\n", global_tid));
     #ifdef KMP_DEBUG
     {
         const char * buff;
         // create format specifiers before the debug output
         buff = __kmp_str_format(
upper=0,stride=1) - JPH June 23, 2009.
         KE_TRACE( 10, ("__kmpc_for_static_init: T#%d return\n", global_tid ) );
         return;
     }
 
     #if OMP_40_ENABLED
     if ( schedtype > kmp_ord_upper ) {
         // we are in DISTRIBUTE construct
         schedtype += kmp_sch_static - kmp_distribute_static;      // AC: convert to usual schedule type
-        tid  = __kmp_threads[ gtid ]->th.th_team->t.t_master_tid;
-        team = __kmp_threads[ gtid ]->th.th_team->t.t_parent;
+        tid  = th->th.th_team->t.t_master_tid;
+        team = th->th.th_team->t.t_parent;
     } else
     #endif
     {
         tid  = __kmp_tid_from_gtid( global_tid );
-        team = __kmp_threads[ gtid ]->th.th_team;
+        team = th->th.th_team;
     }
 
     /* determine if "for" loop is an active worksharing construct */
     if ( team -> t.t_serialized ) {
         /* serialized parallel, each thread executes whole iteration space */
         if( plastiter != NULL )
             *plastiter = TRUE;
         /* leave pupper and plower set to entire iteration space */
case kmp_sch_static_chunked:
         }
     default:
         KMP_ASSERT2( 0, "__kmpc_for_static_init: unknown scheduling type" );
         break;
     }
 
 #if USE_ITT_BUILD
     // Report loop metadata
-    if ( KMP_MASTER_TID(tid) && __itt_metadata_add_ptr && __kmp_forkjoin_frames_mode == 3 ) {
+    if ( KMP_MASTER_TID(tid) && __itt_metadata_add_ptr && __kmp_forkjoin_frames_mode == 3 &&
+#if OMP_40_ENABLED
+        th->th.th_teams_microtask == NULL &&
+#endif
+        team->t.t_active_level == 1 )
+    {
         kmp_uint64 cur_chunk = chunk;
         // Calculate chunk in case it was not specified; it is specified for kmp_sch_static_chunked
         if ( schedtype == kmp_sch_static ) {
             cur_chunk = trip_count / nth + ( ( trip_count % nth ) ? 1 : 0);
         }
         // 0 - "static" schedule
         __kmp_itt_metadata_loop(loc, 0, trip_count, cur_chunk);
     }