[OMPT] Align implementation of reenter frame address to latest (frozen) version of OMPT spec

[protze.joachim]

Summary

The latest OMPT spec changed the semantic of a tasks reenter frame to be the application frame, that will be entered, when the runtime frame drops.
Before it was the last frame in the runtime. This doesn't work for some gcc execution pathes or even clang generated code for :
#omp task if(0)
Since there is no runtime frame between the executed task and the encountering task.

The test case compares exit and reenter addresses against addresses captured in application code

[jlpeyton]

runtime/src/kmp_tasking.c

1357

Can this be changed to NULL to be consistent?

1472

Same here.

[jlpeyton]

Adding Jonas and John to list of reviewers.

[protze.joachim]

Replaced all
_frame = 0;
by
_frame = NULL;
for consistency.

[Hahnfeld]

LGTM from an OMPT perspective

[AndreyChurbanov]

LGTM

[jlpeyton]

LGTM

runtime/src/kmp_csupport.c

// maybe to save thr_state is enough here
     va_start(   ap, microtask );
 
 #if OMPT_SUPPORT
     int tid = __kmp_tid_from_gtid( gtid );
     kmp_info_t *master_th = __kmp_threads[ gtid ];
     kmp_team_t *parent_team = master_th->th.th_team;
     if (ompt_enabled) {
        parent_team->t.t_implicit_task_taskdata[tid].
-           ompt_task_info.frame.reenter_runtime_frame = __builtin_frame_address(0);
+           ompt_task_info.frame.reenter_runtime_frame = __builtin_frame_address(1);
     }
 #endif
 
 #if INCLUDE_SSC_MARKS
     SSC_MARK_FORKING();
 #endif
     __kmp_fork_call( loc, gtid, fork_context_intel,
             argc,

 #endif
     );
 
     va_end( ap );
 
 #if OMPT_SUPPORT
     if (ompt_enabled) {
         parent_team->t.t_implicit_task_taskdata[tid].
-            ompt_task_info.frame.reenter_runtime_frame = 0;
+            ompt_task_info.frame.reenter_runtime_frame = NULL;
     }
 #endif
   }
 }
 
 #if OMP_40_ENABLED
 /*!
 @ingroup PARALLEL
__kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro microtask, ...)
     this_thr->th.th_teams_microtask = microtask;
     this_thr->th.th_teams_level = this_thr->th.th_team->t.t_level; // AC: can be >0 on host
 
 #if OMPT_SUPPORT
     kmp_team_t *parent_team = this_thr->th.th_team;
     int tid = __kmp_tid_from_gtid( gtid );
     if (ompt_enabled) {
         parent_team->t.t_implicit_task_taskdata[tid].
-           ompt_task_info.frame.reenter_runtime_frame = __builtin_frame_address(0);
+           ompt_task_info.frame.reenter_runtime_frame = __builtin_frame_address(1);
     }
 #endif
 
     // check if __kmpc_push_num_teams called, set default number of teams otherwise
     if ( this_thr->th.th_teams_size.nteams == 0 ) {
         __kmp_push_num_teams( loc, gtid, 0, 0 );
     }
     KMP_DEBUG_ASSERT(this_thr->th.th_set_nproc >= 1);
__kmpc_barrier(ident_t *loc, kmp_int32 global_tid)
     if ( __kmp_env_consistency_check ) {
         if ( loc == 0 ) {
             KMP_WARNING( ConstructIdentInvalid ); // ??? What does it mean for the user?
         }; // if
 
         __kmp_check_barrier( global_tid, ct_barrier, loc );
     }
 
+#if OMPT_SUPPORT && OMPT_TRACE
+    ompt_frame_t * ompt_frame;
+    if (ompt_enabled ) {
+        ompt_frame = &( __kmp_threads[ global_tid ] -> th.th_team -> 
+          t.t_implicit_task_taskdata[__kmp_tid_from_gtid(global_tid)].ompt_task_info.frame);
+        ompt_frame->reenter_runtime_frame = __builtin_frame_address(1);
+    }
+#endif
     __kmp_threads[ global_tid ]->th.th_ident = loc;
     // TODO: explicit barrier_wait_id:
     //   this function is called when 'barrier' directive is present or
     //   implicit barrier at the end of a worksharing construct.
     // 1) better to add a per-thread barrier counter to a thread data structure
     // 2) set to 0 when a new team is created
     // 4) no sync is required
 
     __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
+#if OMPT_SUPPORT && OMPT_TRACE
+    if (ompt_enabled ) {
+        ompt_frame->reenter_runtime_frame = NULL;
+    }
+#endif
 }
 
 /* The BARRIER for a MASTER section is always explicit   */
 /*!
 @ingroup WORK_SHARING
 @param loc  source location information.
 @param global_tid  global thread number .
 @return 1 if this thread should execute the <tt>master</tt> block, 0 otherwise.

runtime/src/kmp_gsupport.c

if (ompt_enabled) {
 
         ompt_task_id_t my_ompt_task_id = __ompt_task_id_new(gtid);
 
         // set up lightweight task
         ompt_lw_taskteam_t *lwt = (ompt_lw_taskteam_t *)
             __kmp_allocate(sizeof(ompt_lw_taskteam_t));
         __ompt_lw_taskteam_init(lwt, thr, gtid, (void *) task, ompt_parallel_id);
         lwt->ompt_task_info.task_id = my_ompt_task_id;
-        lwt->ompt_task_info.frame.exit_runtime_frame = 0;
+        lwt->ompt_task_info.frame.exit_runtime_frame = NULL;
         __ompt_lw_taskteam_link(lwt, thr);
 
 #if OMPT_TRACE
         // implicit task callback
         if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)) {
             ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)(
                 ompt_parallel_id, my_ompt_task_id);
         }

 {
     int gtid = __kmp_entry_gtid();
 
 #if OMPT_SUPPORT
     ompt_frame_t *parent_frame;
 
     if (ompt_enabled) {
         parent_frame = __ompt_get_task_frame_internal(0);
-        parent_frame->reenter_runtime_frame = __builtin_frame_address(0);
+        parent_frame->reenter_runtime_frame = __builtin_frame_address(1);
     }
 #endif
 
     MKLOC(loc, "GOMP_parallel_start");
     KA_TRACE(20, ("GOMP_parallel_start: T#%d\n", gtid));
 
     if (__kmpc_ok_to_fork(&loc) && (num_threads != 1)) {
         if (num_threads != 0) {
if (ompt_enabled) {
         parallel_id = team_info->parallel_id;
 
         ompt_task_info_t *task_info = __ompt_get_taskinfo(0);
         serialized_task_id = task_info->task_id;
 
         // Record that we re-entered the runtime system in the implicit
         // task frame representing the parallel region.
         ompt_frame = &task_info->frame;
-        ompt_frame->reenter_runtime_frame = __builtin_frame_address(0);
+        ompt_frame->reenter_runtime_frame = __builtin_frame_address(1);
 
         // unlink if necessary. no-op if there is not a lightweight task.
         ompt_lw_taskteam_t *lwt = __ompt_lw_taskteam_unlink(thr);
         // GOMP allocates/frees lwt since it can't be kept on the stack
         if (lwt) {
            __kmp_free(lwt);
 
 #if OMPT_SUPPORT
            if (ompt_enabled) {
               // Since a lightweight task was destroyed, make sure that the
               // remaining deepest task knows the stack frame where the runtime
               // was reentered.
               ompt_frame = __ompt_get_task_frame_internal(0);
-              ompt_frame->reenter_runtime_frame = __builtin_frame_address(0);
+              ompt_frame->reenter_runtime_frame = __builtin_frame_address(1);
            }
 #endif
         }
     }
 #endif
 
     if (! thr->th.th_team->t.t_serialized) {
         __kmp_run_after_invoked_task(gtid, __kmp_tid_from_gtid(gtid), thr,
           thr->th.th_team);
 
 #if OMPT_SUPPORT
         if (ompt_enabled) {
           // Set reenter frame in parent task, which will become current task
           // in the midst of join. This is needed before the end_parallel callback.
           ompt_frame = __ompt_get_task_frame_internal(1);
-          ompt_frame->reenter_runtime_frame = __builtin_frame_address(0);
+          ompt_frame->reenter_runtime_frame = __builtin_frame_address(1);
         }
 #endif
 
         __kmp_join_call(&loc, gtid
 #if OMPT_SUPPORT
             , fork_context_gnu
 #endif
         );

 #endif
 
         __kmpc_end_serialized_parallel(&loc, gtid);
 
 #if OMPT_SUPPORT
         if (ompt_enabled) {
             // Record that we re-entered the runtime system in the frame that
             // created the parallel region.
-            ompt_frame->reenter_runtime_frame = __builtin_frame_address(0);
+            ompt_frame->reenter_runtime_frame = __builtin_frame_address(1);
 
             if (ompt_callbacks.ompt_callback(ompt_event_parallel_end)) {
                 ompt_task_info_t *task_info = __ompt_get_taskinfo(0);
                 ompt_callbacks.ompt_callback(ompt_event_parallel_end)(
                     parallel_id, task_info->task_id,
                     OMPT_INVOKER(fork_context_gnu));
             }
 

 
 
 #if OMPT_SUPPORT
 
 #define OMPT_LOOP_PRE() \
     ompt_frame_t *parent_frame; \
     if (ompt_enabled) { \
         parent_frame = __ompt_get_task_frame_internal(0); \
-        parent_frame->reenter_runtime_frame = __builtin_frame_address(0); \
+        parent_frame->reenter_runtime_frame = __builtin_frame_address(1); \
     }
 
 
 #define OMPT_LOOP_POST() \
     if (ompt_enabled) { \
         parent_frame->reenter_runtime_frame = NULL; \
     }
 
#endif
 
         __kmpc_omp_task_begin_if0(&loc, gtid, task);
         func(data);
         __kmpc_omp_task_complete_if0(&loc, gtid, task);
 
 #if OMPT_SUPPORT
         if (ompt_enabled) {
             thread->th.ompt_thread_info = oldInfo;
-            taskdata->ompt_task_info.frame.exit_runtime_frame = 0;
+            taskdata->ompt_task_info.frame.exit_runtime_frame = NULL;
         }
 #endif
     }
 
     KA_TRACE(20, ("GOMP_task exit: T#%d\n", gtid));
 }
 
 

 {
     int gtid = __kmp_entry_gtid();
 
 #if OMPT_SUPPORT
     ompt_frame_t *parent_frame;
 
     if (ompt_enabled) {
         parent_frame = __ompt_get_task_frame_internal(0);
-        parent_frame->reenter_runtime_frame = __builtin_frame_address(0);
+        parent_frame->reenter_runtime_frame = __builtin_frame_address(1);
     }
 #endif
 
     MKLOC(loc, "GOMP_parallel_sections_start");
     KA_TRACE(20, ("GOMP_parallel_sections_start: T#%d\n", gtid));
 
     if (__kmpc_ok_to_fork(&loc) && (num_threads != 1)) {
         if (num_threads != 0) {

runtime/src/kmp_runtime.c

#if OMPT_SUPPORT
                                         , exit_runtime_p
 #endif
                                         );
             }
 
 #if OMPT_SUPPORT
             if (ompt_enabled) {
 #if OMPT_TRACE
-                lw_taskteam.ompt_task_info.frame.exit_runtime_frame = 0;
+                lw_taskteam.ompt_task_info.frame.exit_runtime_frame = NULL;
 
                 if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)) {
                     ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)(
                         ompt_parallel_id, ompt_task_id);
                 }
 
                 __ompt_lw_taskteam_unlink(master_th);
                 // reset clear the task id only after unlinking the task

 #if OMPT_SUPPORT
                         , exit_runtime_p
 #endif
                     );
                 }
 
 #if OMPT_SUPPORT
                 if (ompt_enabled) {
-                    lw_taskteam.ompt_task_info.frame.exit_runtime_frame = 0;
+                    lw_taskteam.ompt_task_info.frame.exit_runtime_frame = NULL;
 
 #if OMPT_TRACE
                     if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)) {
                         ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)(
                             ompt_parallel_id, ompt_task_id);
                     }
 #endif
 
#if OMPT_SUPPORT
                         , exit_runtime_p
 #endif
                     );
                 }
 
 #if OMPT_SUPPORT
                 if (ompt_enabled) {
 #if OMPT_TRACE
-                    lw_taskteam.ompt_task_info.frame.exit_runtime_frame = 0;
+                    lw_taskteam.ompt_task_info.frame.exit_runtime_frame = NULL;
 
                     if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)) {
                         ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)(
                             my_parallel_id, my_task_id);
                     }
 #endif
 
                     __ompt_lw_taskteam_unlink(master_th);
#endif /* OMP_40_ENABLED */
         else if ( call_context == fork_context_gnu ) {
 #if OMPT_SUPPORT
             ompt_lw_taskteam_t *lwt = (ompt_lw_taskteam_t *)
                 __kmp_allocate(sizeof(ompt_lw_taskteam_t));
             __ompt_lw_taskteam_init(lwt, master_th, gtid,
                 unwrapped_task, ompt_parallel_id);
 
             lwt->ompt_task_info.task_id = __ompt_task_id_new(gtid);
-            lwt->ompt_task_info.frame.exit_runtime_frame = 0;
+            lwt->ompt_task_info.frame.exit_runtime_frame = NULL;
             __ompt_lw_taskteam_link(lwt, master_th);
 #endif
 
             // we were called from GNU native code
             KA_TRACE( 20, ("__kmp_fork_call: T#%d serial exit\n", gtid ));
             return FALSE;
         }
         else {

 
 #if OMPT_SUPPORT && OMPT_TRACE
     if(ompt_enabled){
         ompt_task_info_t *task_info = __ompt_get_taskinfo(0);
         if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)) {
              ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)(
                parallel_id, task_info->task_id);
         }
-        task_info->frame.exit_runtime_frame = 0;
+        task_info->frame.exit_runtime_frame = NULL;
         task_info->task_id = 0;
     }
 #endif
 
     KF_TRACE( 10, ("__kmp_join_call1: T#%d, this_thread=%p team=%p\n",
                    0, master_th, team ) );
     __kmp_pop_current_task_from_thread( master_th );
 
#endif
                     rc = (*pteam)->t.t_invoke( gtid );
                 }
                 KMP_START_DEVELOPER_EXPLICIT_TIMER(USER_launch_thread_loop);
                 KMP_ASSERT( rc );
 
 #if OMPT_SUPPORT
                 if (ompt_enabled) {
                     /* no frame set while outside task */
-                    task_info->frame.exit_runtime_frame = 0;
+                    task_info->frame.exit_runtime_frame = NULL;
 
                     this_thr->th.ompt_thread_info.state = ompt_state_overhead;
                 }
 #endif
                 KMP_MB();
                 KA_TRACE(20, ("__kmp_launch_thread: T#%d(%d:%d) done microtask = %p\n",
                               gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid), (*pteam)->t.t_pkfn));
             }
             /* join barrier after parallel region */
             __kmp_join_barrier( gtid );
 #if OMPT_SUPPORT && OMPT_TRACE
             if (ompt_enabled) {
                 if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)) {
                     // don't access *pteam here: it may have already been freed
                     // by the master thread behind the barrier (possible race)
                     ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)(
                         my_parallel_id, task_info->task_id);
                 }
-                task_info->frame.exit_runtime_frame = 0;
+                task_info->frame.exit_runtime_frame = NULL;
                 task_info->task_id = 0;
             }
 #endif
         }
     }
     TCR_SYNC_PTR((intptr_t)__kmp_global.g.g_done);
 
 #if OMPT_SUPPORT

runtime/src/kmp_tasking.c

 #if OMP_40_ENABLED
     }
 #endif // OMP_40_ENABLED
 
 
 #if OMPT_SUPPORT
     if (ompt_enabled) {
         thread->th.ompt_thread_info = oldInfo;
-        taskdata->ompt_task_info.frame.exit_runtime_frame = 0;
+        taskdata->ompt_task_info.frame.exit_runtime_frame = NULL;
     }
 #endif
 
 #if OMP_45_ENABLED
     // Proxy tasks are not handled by the runtime
     if ( taskdata->td_flags.proxy != TASK_PROXY )
 #endif
        __kmp_task_finish( gtid, task, current_task );

 kmp_int32
 __kmp_omp_task( kmp_int32 gtid, kmp_task_t * new_task, bool serialize_immediate )
 {
     kmp_taskdata_t * new_taskdata = KMP_TASK_TO_TASKDATA(new_task);
 
 #if OMPT_SUPPORT
     if (ompt_enabled) {
         new_taskdata->ompt_task_info.frame.reenter_runtime_frame =
-            __builtin_frame_address(0);
+            __builtin_frame_address(1);
     }
 #endif
 
     /* Should we execute the new task or queue it?   For now, let's just always try to
        queue it.  If the queue fills up, then we'll execute it.  */
 #if OMP_45_ENABLED
     if ( new_taskdata->td_flags.proxy == TASK_PROXY || __kmp_push_task( gtid, new_task ) == TASK_NOT_PUSHED ) // if cannot defer
 #else
     if ( __kmp_push_task( gtid, new_task ) == TASK_NOT_PUSHED ) // if cannot defer
 #endif
     {                                                           // Execute this task immediately
         kmp_taskdata_t * current_task = __kmp_threads[ gtid ] -> th.th_current_task;
         if ( serialize_immediate )
           new_taskdata -> td_flags.task_serial = 1;
         __kmp_invoke_task( gtid, new_task, current_task );
     }
 
 #if OMPT_SUPPORT
     if (ompt_enabled) {
-        new_taskdata->ompt_task_info.frame.reenter_runtime_frame = 0;
+        new_taskdata->ompt_task_info.frame.reenter_runtime_frame = NULL;
     }
 #endif
 
     return TASK_CURRENT_NOT_QUEUED;
 }
 
 //---------------------------------------------------------------------
 // __kmpc_omp_task: Wrapper around __kmp_omp_task to schedule a non-thread-switchable task from
#if OMPT_SUPPORT && OMPT_TRACE
         ompt_task_id_t my_task_id;
         ompt_parallel_id_t my_parallel_id;
 
         if (ompt_enabled) {
             kmp_team_t *team = thread->th.th_team;
             my_task_id = taskdata->ompt_task_info.task_id;
             my_parallel_id = team->t.ompt_team_info.parallel_id;
 
-            taskdata->ompt_task_info.frame.reenter_runtime_frame = __builtin_frame_address(0);
+            taskdata->ompt_task_info.frame.reenter_runtime_frame = __builtin_frame_address(1);
             if (ompt_callbacks.ompt_callback(ompt_event_taskwait_begin)) {
                 ompt_callbacks.ompt_callback(ompt_event_taskwait_begin)(
                                 my_parallel_id, my_task_id);
             }
         }
 #endif
 
         // Debugger: The taskwait is active. Store location and thread encountered the taskwait.
#endif /* USE_ITT_BUILD */
         taskdata->td_taskwait_thread = - taskdata->td_taskwait_thread;
 
 #if OMPT_SUPPORT && OMPT_TRACE
         if (ompt_enabled) {
             if (ompt_callbacks.ompt_callback(ompt_event_taskwait_end)) {
                 ompt_callbacks.ompt_callback(ompt_event_taskwait_end)(
                                 my_parallel_id, my_task_id);
             }
-            taskdata->ompt_task_info.frame.reenter_runtime_frame = 0;
+            taskdata->ompt_task_info.frame.reenter_runtime_frame = NULL;
         }
 #endif
     }
 
     KA_TRACE(10, ("__kmpc_omp_taskwait(exit): T#%d task %p finished waiting, "
                   "returning TASK_CURRENT_NOT_QUEUED\n", gtid, taskdata) );
 
     return TASK_CURRENT_NOT_QUEUED;

runtime/src/ompt-specific.c

 void
 __ompt_lw_taskteam_init(ompt_lw_taskteam_t *lwt, kmp_info_t *thr,
                         int gtid, void *microtask,
                         ompt_parallel_id_t ompt_pid)
 {
     lwt->ompt_team_info.parallel_id = ompt_pid;
     lwt->ompt_team_info.microtask = microtask;
     lwt->ompt_task_info.task_id = 0;
-    lwt->ompt_task_info.frame.reenter_runtime_frame = 0;
-    lwt->ompt_task_info.frame.exit_runtime_frame = 0;
+    lwt->ompt_task_info.frame.reenter_runtime_frame = NULL;
+    lwt->ompt_task_info.frame.exit_runtime_frame = NULL;
     lwt->ompt_task_info.function = NULL;
     lwt->parent = 0;
 }
 
 
 void
 __ompt_lw_taskteam_link(ompt_lw_taskteam_t *lwt,  kmp_info_t *thr)
 {

runtime/test/ompt/parallel/nested.c

 // RUN: %libomp-compile-and-run | FileCheck %s
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck --check-prefix=THREADS %s
 // REQUIRES: ompt
 #include "callback.h"
 #include <omp.h>
 
 int main()
 {
   omp_set_nested(1);
+  print_frame(0);
 
   #pragma omp parallel num_threads(4)
   {
+    print_frame(1);
     print_ids(0);
     print_ids(1);
+    print_frame(0);
     #pragma omp parallel num_threads(4)
     {
+      print_frame(1);
       print_ids(0);
       print_ids(1);
       print_ids(2);
+      print_frame(0);
+      #pragma omp barrier
     }
   }
 
   // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
   // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-9]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{0x[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-9]+]], requested_team_size=4, parallel_function=0x{{[0-f]+}}, invoker=[[PARALLEL_INVOKER:.+]]
 
   // CHECK-DAG: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
   // CHECK-DAG: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
 
   // Note that we cannot ensure that the worker threads have already called barrier_end and implicit_task_end before parallel_end!
 
   // CHECK-DAG: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
   // CHECK-DAG: {{^}}[[THREAD_ID]]: ompt_event_barrier_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
 
   // CHECK-DAG: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
   // CHECK-DAG: {{^}}[[THREAD_ID]]: ompt_event_barrier_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
 
   // CHECK-DAG: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
   // CHECK-DAG: {{^}}[[THREAD_ID]]: ompt_event_barrier_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
 
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_parallel_end: parallel_id=[[PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], invoker=[[PARALLEL_INVOKER]]
 
 
-  // THREADS: 0: NULL_POINTER=[[NULL:.*$]]
-  // THREADS: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-9]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{0x[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-9]+]], requested_team_size=4, parallel_function=0x{{[0-f]+}}, invoker=[[PARALLEL_INVOKER:.+]]
+  // THREADS: {{^}}0: NULL_POINTER=[[NULL:.*$]]
+  // THREADS: {{^}}[[MASTER_ID:[0-9]+]]: __builtin_frame_address(0)=[[MAIN_REENTER:0x[0-f]+]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-9]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter=[[MAIN_REENTER]], parallel_id=[[PARALLEL_ID:[0-9]+]], requested_team_size=4, parallel_function=0x{{[0-f]+}}, invoker=[[PARALLEL_INVOKER:.+]]
 
   // nested parallel masters
   // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
-  // THREADS: {{^}}[[MASTER_ID]]: level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[MASTER_ID]]: level 1: parallel_id=0, task_id=[[PARENT_TASK_ID]]
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit={{0x[0-f]+}}, parent_task_frame.reenter={{0x[0-f]+}}, parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], requested_team_size=4, parallel_function=[[NESTED_PARALLEL_FUNCTION:0x[0-f]+]], invoker=[[PARALLEL_INVOKER]]
+  // THREADS: {{^}}[[MASTER_ID]]: __builtin_frame_address(1)=[[EXIT:0x[0-f]+]]
+  // THREADS: {{^}}[[MASTER_ID]]: level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT]], reenter_frame=[[NULL]]
+  // THREADS: {{^}}[[MASTER_ID]]: level 1: parallel_id=0, task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[MAIN_REENTER]]
+  // THREADS: {{^}}[[MASTER_ID]]: __builtin_frame_address(0)=[[REENTER:0x[0-f]+]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit=[[EXIT]], parent_task_frame.reenter=[[REENTER]], parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], requested_team_size=4, parallel_function=[[NESTED_PARALLEL_FUNCTION:0x[0-f]+]], invoker=[[PARALLEL_INVOKER]]
   // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-9]+]]
-  // THREADS: {{^}}[[MASTER_ID]]: level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[MASTER_ID]]: level 1: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[MASTER_ID]]: level 2: parallel_id=0, task_id=[[PARENT_TASK_ID]]
+  // THREADS: {{^}}[[MASTER_ID]]: __builtin_frame_address(1)=[[NESTED_EXIT:0x[0-f]+]]
+  // THREADS: {{^}}[[MASTER_ID]]: level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]], exit_frame=[[NESTED_EXIT]], reenter_frame=[[NULL]]
+  // THREADS: {{^}}[[MASTER_ID]]: level 1: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT]], reenter_frame=[[REENTER]]
+  // THREADS: {{^}}[[MASTER_ID]]: level 2: parallel_id=0, task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[MAIN_REENTER]]
+  // THREADS: {{^}}[[MASTER_ID]]: __builtin_frame_address(0)=[[NESTED_REENTER:0x[0-f]+]]
   // THREADS-NOT: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[MASTER_ID]]: ompt_event_barrier_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[MASTER_ID]]: ompt_event_barrier_end: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_end: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], invoker=[[PARALLEL_INVOKER]]
   // THREADS-NOT: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[MASTER_ID]]: ompt_event_barrier_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[MASTER_ID]]: ompt_event_barrier_end: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]