[OpenMP][OMPT] Fix OMPT address handling after D92197

[protze.joachim]

Summary

The split of interfaces in D92197 breaks some of the OMPT tests by storing the wrong return or frame address.
This patch also introduces KMP_DEBUG_ASSERTIONS to verify expected state of the stored return address.
The return address is now managed in a thread-local stack in order to avoid repeated setting and unsetting the address.

openmp/runtime/src/kmp_barrier.cpp

/* After performing this write, a worker thread may not assume that the team
        time. */
     ANNOTATE_BARRIER_BEGIN(this_thr);
     kmp_flag_64<> flag(&thr_bar->b_arrived, other_threads[0]);
     flag.release();
   } else {
     kmp_balign_team_t *team_bar = &team->t.t_bar[bt];
     int nproc = this_thr->th.th_team_nproc;
     int i;
+    OMPT_REDUCTION_DECL_IF(this_thr, gtid, reduce);
     // Don't have to worry about sleep bit here or atomic since team setting
     kmp_uint64 new_state = team_bar->b_arrived + KMP_BARRIER_STATE_BUMP;
 
     // Collect all the worker team member threads.
     for (i = 1; i < nproc; ++i) {
 #if KMP_CACHE_MANAGE
       // Prefetch next thread's arrived count
       if (i + 1 < nproc)
#if USE_ITT_BUILD && USE_ITT_NOTIFY
       }
 #endif
       if (reduce) {
         KA_TRACE(100,
                  ("__kmp_linear_barrier_gather: T#%d(%d:%d) += T#%d(%d:%d)\n",
                   gtid, team->t.t_id, tid, __kmp_gtid_from_tid(i, team),
                   team->t.t_id, i));
         ANNOTATE_REDUCE_AFTER(reduce);
-        OMPT_REDUCTION_DECL(this_thr, gtid);
         OMPT_REDUCTION_BEGIN;
         (*reduce)(this_thr->th.th_local.reduce_data,
                   other_threads[i]->th.th_local.reduce_data);
         OMPT_REDUCTION_END;
         ANNOTATE_REDUCE_BEFORE(reduce);
         ANNOTATE_REDUCE_BEFORE(&team->t.t_bar);
       }
     }

 #endif
   // Perform tree gather to wait until all threads have arrived; reduce any
   // required data as we go
   child_tid = (tid << branch_bits) + 1;
   if (child_tid < nproc) {
     // Parent threads wait for all their children to arrive
     new_state = team->t.t_bar[bt].b_arrived + KMP_BARRIER_STATE_BUMP;
     child = 1;
+    OMPT_REDUCTION_DECL_IF(this_thr, gtid, reduce);
     do {
       kmp_info_t *child_thr = other_threads[child_tid];
       kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb;
 #if KMP_CACHE_MANAGE
       // Prefetch next thread's arrived count
       if (child + 1 <= branch_factor && child_tid + 1 < nproc)
         KMP_CACHE_PREFETCH(
             &other_threads[child_tid + 1]->th.th_bar[bt].bb.b_arrived);
#if USE_ITT_BUILD && USE_ITT_NOTIFY
       }
 #endif
       if (reduce) {
         KA_TRACE(100,
                  ("__kmp_tree_barrier_gather: T#%d(%d:%d) += T#%d(%d:%u)\n",
                   gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
                   team->t.t_id, child_tid));
         ANNOTATE_REDUCE_AFTER(reduce);
-        OMPT_REDUCTION_DECL(this_thr, gtid);
         OMPT_REDUCTION_BEGIN;
         (*reduce)(this_thr->th.th_local.reduce_data,
                   child_thr->th.th_local.reduce_data);
         OMPT_REDUCTION_END;
         ANNOTATE_REDUCE_BEFORE(reduce);
         ANNOTATE_REDUCE_BEFORE(&team->t.t_bar);
       }
       child++;
#if USE_ITT_BUILD && USE_ITT_NOTIFY
   if (__kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 2) {
     this_thr->th.th_bar_arrive_time = this_thr->th.th_bar_min_time =
         __itt_get_timestamp();
   }
 #endif
   /* Perform a hypercube-embedded tree gather to wait until all of the threads
      have arrived, and reduce any required data as we go.  */
   kmp_flag_64<> p_flag(&thr_bar->b_arrived);
+  OMPT_REDUCTION_DECL_IF(this_thr, gtid, reduce);
   for (level = 0, offset = 1; offset < num_threads;
        level += branch_bits, offset <<= branch_bits) {
     kmp_uint32 child;
     kmp_uint32 child_tid;
 
     if (((tid >> level) & (branch_factor - 1)) != 0) {
       kmp_int32 parent_tid = tid & ~((1 << (level + branch_bits)) - 1);
 
#if USE_ITT_BUILD && USE_ITT_NOTIFY
       }
 #endif
       if (reduce) {
         KA_TRACE(100,
                  ("__kmp_hyper_barrier_gather: T#%d(%d:%d) += T#%d(%d:%u)\n",
                   gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
                   team->t.t_id, child_tid));
         ANNOTATE_REDUCE_AFTER(reduce);
-        OMPT_REDUCTION_DECL(this_thr, gtid);
         OMPT_REDUCTION_BEGIN;
         (*reduce)(this_thr->th.th_local.reduce_data,
                   child_thr->th.th_local.reduce_data);
         OMPT_REDUCTION_END;
         ANNOTATE_REDUCE_BEFORE(reduce);
         ANNOTATE_REDUCE_BEFORE(&team->t.t_bar);
       }
     }

openmp/runtime/src/kmp_cancel.cpp

#if OMPT_SUPPORT && OMPT_OPTIONAL
             if (cncl_kind == cancel_parallel)
               type = ompt_cancel_parallel;
             else if (cncl_kind == cancel_loop)
               type = ompt_cancel_loop;
             else if (cncl_kind == cancel_sections)
               type = ompt_cancel_sections;
             ompt_callbacks.ompt_callback(ompt_callback_cancel)(
                 task_data, type | ompt_cancel_activated,
-                OMPT_GET_RETURN_ADDRESS(0));
+                OMPT_LOAD_RETURN_ADDRESS(gtid));
           }
 #endif // OMPT_SUPPORT && OMPT_OPTIONAL
           return 1 /* true */;
         }
         break;
       }
     case cancel_taskgroup:
       // cancellation requests for a task group

 // have one that matches the current request -> cancel
 #if OMPT_SUPPORT && OMPT_OPTIONAL
             if (ompt_enabled.ompt_callback_cancel) {
               ompt_data_t *task_data;
               __ompt_get_task_info_internal(0, NULL, &task_data, NULL, NULL,
                                             NULL);
               ompt_callbacks.ompt_callback(ompt_callback_cancel)(
                   task_data, ompt_cancel_taskgroup | ompt_cancel_activated,
-                  OMPT_GET_RETURN_ADDRESS(0));
+                  OMPT_LOAD_RETURN_ADDRESS(gtid));
             }
 #endif
             return 1 /* true */;
           }
         } else {
           // TODO: what needs to happen here?
           // the specification disallows cancellation w/o taskgroups
           // so we might do anything here, let's abort for now
           KMP_ASSERT(0 /* false */);
         }
       }
       break;
     default:
       KMP_ASSERT(0 /* false */);
     }
   }
 
   // ICV OMP_CANCELLATION=false, so we ignored this cancel request
   KMP_DEBUG_ASSERT(!__kmp_omp_cancellation);
   return 0 /* false */;
 }
 
 kmp_int32 __kmpc_cancel(ident_t *loc_ref, kmp_int32 gtid, kmp_int32 cncl_kind) {
+#if OMPT_SUPPORT && OMPT_OPTIONAL
+  OMPT_STORE_RETURN_ADDRESS(gtid);
+#endif
   return __kmp_aux_cancel(loc_ref, gtid, cncl_kind);
 }
 
 /*!
 @ingroup CANCELLATION
 @param loc_ref location of the original task directive
 @param gtid Global thread ID of encountering thread
 @param cncl_kind Cancellation kind (parallel, for, sections, taskgroup)
#if OMPT_SUPPORT && OMPT_OPTIONAL
               if (cncl_kind == cancel_parallel)
                 type = ompt_cancel_parallel;
               else if (cncl_kind == cancel_loop)
                 type = ompt_cancel_loop;
               else if (cncl_kind == cancel_sections)
                 type = ompt_cancel_sections;
               ompt_callbacks.ompt_callback(ompt_callback_cancel)(
                   task_data, type | ompt_cancel_detected,
-                  OMPT_GET_RETURN_ADDRESS(0));
+                  OMPT_LOAD_RETURN_ADDRESS(gtid));
             }
 #endif
             return 1 /* true */;
           }
           KMP_ASSERT(0 /* false */);
         } else {
           // we do not have a cancellation request pending, so we just
           // ignore this cancellation point

 #if OMPT_SUPPORT && OMPT_OPTIONAL
           if (ompt_enabled.ompt_callback_cancel &&
               !!taskgroup->cancel_request) {
             ompt_data_t *task_data;
             __ompt_get_task_info_internal(0, NULL, &task_data, NULL, NULL,
                                           NULL);
             ompt_callbacks.ompt_callback(ompt_callback_cancel)(
                 task_data, ompt_cancel_taskgroup | ompt_cancel_detected,
-                OMPT_GET_RETURN_ADDRESS(0));
+                OMPT_LOAD_RETURN_ADDRESS(gtid));
           }
 #endif
           return !!taskgroup->cancel_request;
         } else {
           // if a cancellation point is encountered by a task that does not
           // belong to a taskgroup, it is OK to ignore it
           return 0 /* false */;
         }
       }
     default:
       KMP_ASSERT(0 /* false */);
     }
   }
 
   // ICV OMP_CANCELLATION=false, so we ignore the cancellation point
   KMP_DEBUG_ASSERT(!__kmp_omp_cancellation);
   return 0 /* false */;
 }
 
 kmp_int32 __kmpc_cancellationpoint(ident_t *loc_ref, kmp_int32 gtid,
                                    kmp_int32 cncl_kind) {
+#if OMPT_SUPPORT && OMPT_OPTIONAL
+  OMPT_STORE_RETURN_ADDRESS(gtid);
+#endif
   return __kmp_aux_cancellationpoint(loc_ref, gtid, cncl_kind);
 }
+
 /*!
 @ingroup CANCELLATION
 @param loc_ref location of the original task directive
 @param gtid Global thread ID of encountering thread
 
 @return returns true if a matching cancellation request has been flagged in the
 RTL and the encountering thread has to cancel..
 

openmp/runtime/src/kmp_csupport.cpp

 #if OMPT_SUPPORT
   if (ompt_enabled.enabled)
     this_thr->th.ompt_thread_info.state =
         ((this_thr->th.th_team_serialized) ? ompt_state_work_serial
                                            : ompt_state_work_parallel);
 #endif
 }
 void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 global_tid) {
+#if OMPT_SUPPORT
+  OMPT_STORE_RETURN_ADDRESS(global_tid);
+#endif
   __kmp_aux_end_serialized_parallel(loc, global_tid);
 }
 
 /*!
 @ingroup SYNCHRONIZATION
 @param loc  source location information.
 
 Execute <tt>flush</tt>. This is implemented as a full memory fence. (Though
void __kmp_aux_barrier(ident_t *loc, kmp_int32 global_tid) {
 
   if (__kmp_env_consistency_check) {
     if (loc == 0) {
       KMP_WARNING(ConstructIdentInvalid); // ??? What does it mean for the user?
     }
     __kmp_check_barrier(global_tid, ct_barrier, loc);
   }
 
-#if OMPT_SUPPORT
-  ompt_frame_t *ompt_frame;
-  if (ompt_enabled.enabled) {
-    __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
-    if (ompt_frame->enter_frame.ptr == NULL)
-      ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
-  }
-  OMPT_STORE_RETURN_ADDRESS(global_tid);
-#endif
   __kmp_threads[global_tid]->th.th_ident = loc;
   // TODO: explicit barrier_wait_id:
   //   this function is called when 'barrier' directive is present or
   //   implicit barrier at the end of a worksharing construct.
   // 1) better to add a per-thread barrier counter to a thread data structure
   // 2) set to 0 when a new team is created
   // 4) no sync is required
 
   __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
-#if OMPT_SUPPORT && OMPT_OPTIONAL
-  if (ompt_enabled.enabled) {
-    ompt_frame->enter_frame = ompt_data_none;
-  }
-#endif
 }
 
 void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid) {
   __kmp_assert_valid_gtid(global_tid);
+#if OMPT_SUPPORT
+  ompt_frame_t *ompt_frame;
+  if (ompt_enabled.enabled) {
+    __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
+    if (ompt_frame->enter_frame.ptr == NULL)
+      ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
+  }
+  OMPT_STORE_RETURN_ADDRESS(global_tid);
+#endif
   __kmp_aux_barrier(loc, global_tid);
+#if OMPT_SUPPORT && OMPT_OPTIONAL
+  if (ompt_enabled.enabled) {
+    ompt_frame->enter_frame = ompt_data_none;
+  }
+#endif
 }
 
 /* The BARRIER for a MASTER section is always explicit   */
 /*!
 @ingroup WORK_SHARING
 @param loc  source location information.
 @param global_tid  global thread number .
 @return 1 if this thread should execute the <tt>master</tt> block, 0 otherwise.

 #endif /* USE_ITT_BUILD */
 
   th = __kmp_threads[gtid];
 
 #if OMPT_SUPPORT && OMPT_OPTIONAL
   kmp_team_t *team;
   ompt_wait_id_t lck;
   void *codeptr_ra;
-  OMPT_STORE_RETURN_ADDRESS(gtid);
   if (ompt_enabled.enabled) {
     team = __kmp_team_from_gtid(gtid);
     lck = (ompt_wait_id_t)(uintptr_t)&team->t.t_ordered.dt.t_value;
     /* OMPT state update */
     th->th.ompt_thread_info.wait_id = lck;
     th->th.ompt_thread_info.state = ompt_state_wait_ordered;
 
     /* OMPT event callback */
#if OMPT_SUPPORT && OMPT_OPTIONAL
   }
 #endif
 
 #if USE_ITT_BUILD
   __kmp_itt_ordered_start(gtid);
 #endif /* USE_ITT_BUILD */
 }
 void __kmpc_ordered(ident_t *loc, kmp_int32 gtid) {
+#if OMPT_SUPPORT && OMPT_OPTIONAL
+  OMPT_STORE_RETURN_ADDRESS(gtid);
+#endif
   __kmp_aux_ordered(loc, gtid);
 }
 
 /*!
 @ingroup WORK_SHARING
 @param loc  source location information.
 @param gtid  global thread number.
 
#endif /* USE_ITT_BUILD */
   th = __kmp_threads[gtid];
 
   if (th->th.th_dispatch->th_dxo_fcn != 0)
     (*th->th.th_dispatch->th_dxo_fcn)(&gtid, &cid, loc);
   else
     __kmp_parallel_dxo(&gtid, &cid, loc);
 
 #if OMPT_SUPPORT && OMPT_OPTIONAL
-  OMPT_STORE_RETURN_ADDRESS(gtid);
   if (ompt_enabled.ompt_callback_mutex_released) {
     ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
         ompt_mutex_ordered,
         (ompt_wait_id_t)(uintptr_t)&__kmp_team_from_gtid(gtid)
             ->t.t_ordered.dt.t_value,
         OMPT_LOAD_RETURN_ADDRESS(gtid));
   }
 #endif
 }
 void __kmpc_end_ordered(ident_t *loc, kmp_int32 gtid) {
+#if OMPT_SUPPORT && OMPT_OPTIONAL
+  OMPT_STORE_RETURN_ADDRESS(gtid);
+#endif
   __kmp_aux_end_ordered(loc, gtid);
 }
 
 #if KMP_USE_DYNAMIC_LOCK
 
 static __forceinline void
 __kmp_init_indirect_csptr(kmp_critical_name *crit, ident_t const *loc,
                           kmp_int32 gtid, kmp_indirect_locktag_t tag) {

 associated with the critical section, or some other suitably unique value.
 
 Enter code protected by a `critical` construct.
 This function blocks until the executing thread can enter the critical section.
 */
 void __kmp_aux_critical(ident_t *loc, kmp_int32 global_tid,
                         kmp_critical_name *crit) {
 #if KMP_USE_DYNAMIC_LOCK
-#if OMPT_SUPPORT && OMPT_OPTIONAL
-  OMPT_STORE_RETURN_ADDRESS(global_tid);
-#endif // OMPT_SUPPORT
   __kmp_aux_critical_with_hint(loc, global_tid, crit, omp_lock_hint_none);
 #else
   KMP_COUNT_BLOCK(OMP_CRITICAL);
 #if OMPT_SUPPORT && OMPT_OPTIONAL
   ompt_state_t prev_state = ompt_state_undefined;
   ompt_thread_info_t ti;
 #endif
   kmp_user_lock_p lck;

 // team we have to check this even if we are in a serialized team.
 // also, even if we are the uber thread, we still have to conduct the lock,
 // as we have to contend with sibling threads.
 
 #if USE_ITT_BUILD
   __kmp_itt_critical_acquiring(lck);
 #endif /* USE_ITT_BUILD */
 #if OMPT_SUPPORT && OMPT_OPTIONAL
-  OMPT_STORE_RETURN_ADDRESS(gtid);
   void *codeptr_ra = NULL;
   if (ompt_enabled.enabled) {
     ti = __kmp_threads[global_tid]->th.ompt_thread_info;
     /* OMPT state update */
     prev_state = ti.state;
     ti.wait_id = (ompt_wait_id_t)(uintptr_t)lck;
     ti.state = ompt_state_wait_critical;
 
#endif
 
   KMP_PUSH_PARTITIONED_TIMER(OMP_critical);
   KA_TRACE(15, ("__kmpc_critical: done T#%d\n", global_tid));
 #endif // KMP_USE_DYNAMIC_LOCK
 }
 
 void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
                      kmp_critical_name *crit) {
+#if OMPT_SUPPORT && OMPT_OPTIONAL
+  OMPT_STORE_RETURN_ADDRESS(global_tid);
+#endif // OMPT_SUPPORT
   __kmp_aux_critical(loc, global_tid, crit);
 }
 
 #if KMP_USE_DYNAMIC_LOCK
 
 // Converts the given hint to an internal lock implementation
 static __forceinline kmp_dyna_lockseq_t __kmp_map_hint_to_lock(uintptr_t hint) {
 #if KMP_USE_TSX
void __kmp_aux_critical_with_hint(ident_t *loc, kmp_int32 global_tid,
                                   kmp_critical_name *crit, uint32_t hint) {
   KMP_COUNT_BLOCK(OMP_CRITICAL);
   kmp_user_lock_p lck;
 #if OMPT_SUPPORT && OMPT_OPTIONAL
   ompt_state_t prev_state = ompt_state_undefined;
   ompt_thread_info_t ti;
   // This is the case, if called from __kmpc_critical:
   void *codeptr = OMPT_LOAD_RETURN_ADDRESS(global_tid);
-  if (!codeptr)
-    codeptr = OMPT_GET_RETURN_ADDRESS(0);
 #endif
 
   KC_TRACE(10, ("__kmpc_critical: called T#%d\n", global_tid));
   __kmp_assert_valid_gtid(global_tid);
 
   kmp_dyna_lock_t *lk = (kmp_dyna_lock_t *)crit;
   // Check if it is initialized.
   KMP_PUSH_PARTITIONED_TIMER(OMP_critical_wait);

 #endif
 
   KMP_PUSH_PARTITIONED_TIMER(OMP_critical);
   KA_TRACE(15, ("__kmpc_critical: done T#%d\n", global_tid));
 } // __kmpc_critical_with_hint
 
 void __kmpc_critical_with_hint(ident_t *loc, kmp_int32 global_tid,
                                kmp_critical_name *crit, uint32_t hint) {
+#if OMPT_SUPPORT && OMPT_OPTIONAL
+  OMPT_STORE_RETURN_ADDRESS(global_tid);
+#endif
   __kmp_aux_critical_with_hint(loc, global_tid, crit, hint);
 }
 
 #endif // KMP_USE_DYNAMIC_LOCK
 
 /*!
 @ingroup WORK_SHARING
 @param loc  source location information.
#endif /* USE_ITT_BUILD */
   // section directive.
   __kmp_release_user_lock_with_checks(lck, global_tid);
 
 #endif // KMP_USE_DYNAMIC_LOCK
 
 #if OMPT_SUPPORT && OMPT_OPTIONAL
   /* OMPT release event triggers after lock is released; place here to trigger
    * for all #if branches */
-  OMPT_STORE_RETURN_ADDRESS(global_tid);
   if (ompt_enabled.ompt_callback_mutex_released) {
     ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
         ompt_mutex_critical, (ompt_wait_id_t)(uintptr_t)lck,
-        OMPT_LOAD_RETURN_ADDRESS(0));
+        OMPT_LOAD_RETURN_ADDRESS(global_tid));
   }
 #endif
 
   KMP_POP_PARTITIONED_TIMER();
   KA_TRACE(15, ("__kmpc_end_critical: done T#%d\n", global_tid));
 }
 
 void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
                          kmp_critical_name *crit) {
+#if OMPT_SUPPORT && OMPT_OPTIONAL
+  OMPT_STORE_RETURN_ADDRESS(global_tid);
+#endif
   __kmp_aux_end_critical(loc, global_tid, crit);
 }
 
 /*!
 @ingroup SYNCHRONIZATION
 @param loc source location information
 @param global_tid thread id.
 @return one if the thread should execute the master block, zero otherwise

 #endif
   __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
 
   if (!didit)
     (*cpy_func)(cpy_data, *data_ptr);
 
 // Consider next barrier a user-visible barrier for barrier region boundaries
 // Nesting checks are already handled by the single construct checks
-  {
-#if OMPT_SUPPORT
-    OMPT_STORE_RETURN_ADDRESS(gtid);
-#endif
 #if USE_ITT_NOTIFY
   __kmp_threads[gtid]->th.th_ident = loc; // TODO: check if it is needed (e.g.
 // tasks can overwrite the location)
 #endif
   __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
-#if OMPT_SUPPORT && OMPT_OPTIONAL
-  if (ompt_enabled.enabled) {
-    ompt_frame->enter_frame = ompt_data_none;
-  }
-#endif
-  }
 }
 
 /* -------------------------------------------------------------------------- */
 
 #define INIT_LOCK __kmp_init_user_lock_with_checks
 #define INIT_NESTED_LOCK __kmp_init_nested_user_lock_with_checks
 #define ACQUIRE_LOCK __kmp_acquire_user_lock_with_checks
 #define ACQUIRE_LOCK_TIMED __kmp_acquire_user_lock_with_checks_timed
__kmpc_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars,
   // do not call __kmp_serial_initialize(), it will be called by
   // __kmp_parallel_initialize() if needed
   // possible detection of false-positive race by the threadchecker ???
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
 
   __kmp_resume_if_soft_paused();
 
+#if OMPT_SUPPORT && OMPT_OPTIONAL
+  OMPT_STORE_RETURN_ADDRESS(global_tid);
+#endif
 // check correctness of reduce block nesting
 #if KMP_USE_DYNAMIC_LOCK
   if (__kmp_env_consistency_check)
     __kmp_push_sync(global_tid, ct_reduce, loc, NULL, 0);
 #else
   if (__kmp_env_consistency_check)
     __kmp_push_sync(global_tid, ct_reduce, loc, NULL);
 #endif
#if OMPT_SUPPORT
     // will provide the barrier events
     //         so we set-up the necessary frame/return addresses.
     ompt_frame_t *ompt_frame;
     if (ompt_enabled.enabled) {
       __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
       if (ompt_frame->enter_frame.ptr == NULL)
         ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
     }
-    OMPT_STORE_RETURN_ADDRESS(global_tid);
 #endif
 #if USE_ITT_NOTIFY
     __kmp_threads[global_tid]->th.th_ident = loc;
 #endif
     retval =
         __kmp_barrier(UNPACK_REDUCTION_BARRIER(packed_reduction_method),
                       global_tid, FALSE, reduce_size, reduce_data, reduce_func);
     retval = (retval != 0) ? (0) : (1);

 */
 void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
                               kmp_critical_name *lck) {
 
   PACKED_REDUCTION_METHOD_T packed_reduction_method;
 
   KA_TRACE(10, ("__kmpc_end_reduce_nowait() enter: called T#%d\n", global_tid));
   __kmp_assert_valid_gtid(global_tid);
+#if OMPT_SUPPORT && OMPT_OPTIONAL
+  OMPT_STORE_RETURN_ADDRESS(global_tid);
+#endif
 
   packed_reduction_method = __KMP_GET_REDUCTION_METHOD(global_tid);
 
   OMPT_REDUCTION_DECL(__kmp_thread_from_gtid(global_tid), global_tid);
 
   if (packed_reduction_method == critical_reduce_block) {
 
     __kmp_end_critical_section_reduce_block(loc, global_tid, lck);
kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars,
   // do not call __kmp_serial_initialize(), it will be called by
   // __kmp_parallel_initialize() if needed
   // possible detection of false-positive race by the threadchecker ???
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
 
   __kmp_resume_if_soft_paused();
 
+#if OMPT_SUPPORT && OMPT_OPTIONAL
+  OMPT_STORE_RETURN_ADDRESS(global_tid);
+#endif
 // check correctness of reduce block nesting
 #if KMP_USE_DYNAMIC_LOCK
   if (__kmp_env_consistency_check)
     __kmp_push_sync(global_tid, ct_reduce, loc, NULL, 0);
 #else
   if (__kmp_env_consistency_check)
     __kmp_push_sync(global_tid, ct_reduce, loc, NULL);
 #endif

 // tool (it's a terminating barrier on constructs if NOWAIT not specified)
 #if OMPT_SUPPORT
     ompt_frame_t *ompt_frame;
     if (ompt_enabled.enabled) {
       __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
       if (ompt_frame->enter_frame.ptr == NULL)
         ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
     }
-    OMPT_STORE_RETURN_ADDRESS(global_tid);
 #endif
 #if USE_ITT_NOTIFY
     __kmp_threads[global_tid]->th.th_ident =
         loc; // needed for correct notification of frames
 #endif
     retval =
         __kmp_barrier(UNPACK_REDUCTION_BARRIER(packed_reduction_method),
                       global_tid, TRUE, reduce_size, reduce_data, reduce_func);
void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
   PACKED_REDUCTION_METHOD_T packed_reduction_method;
   kmp_info_t *th;
   kmp_team_t *team;
   int teams_swapped = 0, task_state;
 
   KA_TRACE(10, ("__kmpc_end_reduce() enter: called T#%d\n", global_tid));
   __kmp_assert_valid_gtid(global_tid);
 
+#if OMPT_SUPPORT && OMPT_OPTIONAL
+  OMPT_STORE_RETURN_ADDRESS(global_tid);
+#endif
   th = __kmp_thread_from_gtid(global_tid);
   teams_swapped = __kmp_swap_teams_for_teams_reduction(th, &team, &task_state);
 
   packed_reduction_method = __KMP_GET_REDUCTION_METHOD(global_tid);
 
   // this barrier should be visible to a customer and to the threading profile
   // tool (it's a terminating barrier on constructs if NOWAIT not specified)
   OMPT_REDUCTION_DECL(th, global_tid);
 
   if (packed_reduction_method == critical_reduce_block) {
     __kmp_end_critical_section_reduce_block(loc, global_tid, lck);
 
     OMPT_REDUCTION_END;
 
 // TODO: implicit barrier: should be exposed
 #if OMPT_SUPPORT
     ompt_frame_t *ompt_frame;
     if (ompt_enabled.enabled) {
       __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
       if (ompt_frame->enter_frame.ptr == NULL)
         ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
     }
-    OMPT_STORE_RETURN_ADDRESS(global_tid);
 #endif
 #if USE_ITT_NOTIFY
     __kmp_threads[global_tid]->th.th_ident = loc;
 #endif
     __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
 #if OMPT_SUPPORT && OMPT_OPTIONAL
     if (ompt_enabled.enabled) {
       ompt_frame->enter_frame = ompt_data_none;

 // TODO: implicit barrier: should be exposed
 #if OMPT_SUPPORT
     ompt_frame_t *ompt_frame;
     if (ompt_enabled.enabled) {
       __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
       if (ompt_frame->enter_frame.ptr == NULL)
         ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
     }
-    OMPT_STORE_RETURN_ADDRESS(global_tid);
 #endif
 #if USE_ITT_NOTIFY
     __kmp_threads[global_tid]->th.th_ident = loc;
 #endif
     __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
 #if OMPT_SUPPORT && OMPT_OPTIONAL
     if (ompt_enabled.enabled) {
       ompt_frame->enter_frame = ompt_data_none;
     }
 #endif
 
   } else if (packed_reduction_method == atomic_reduce_block) {
 
 #if OMPT_SUPPORT
     ompt_frame_t *ompt_frame;
     if (ompt_enabled.enabled) {
       __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
       if (ompt_frame->enter_frame.ptr == NULL)
         ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
     }
-    OMPT_STORE_RETURN_ADDRESS(global_tid);
 #endif
 // TODO: implicit barrier: should be exposed
 #if USE_ITT_NOTIFY
     __kmp_threads[global_tid]->th.th_ident = loc;
 #endif
     __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
 #if OMPT_SUPPORT && OMPT_OPTIONAL
     if (ompt_enabled.enabled) {

openmp/runtime/src/kmp_dispatch.cpp

 @param p_st   Pointer to the stride for the next chunk of work
 @return one if there is work to be done, zero otherwise
 
 Get the next dynamically allocated chunk of work for this thread.
 If there is no more work, then the lb,ub and stride need not be modified.
 */
 int __kmpc_dispatch_next_4(ident_t *loc, kmp_int32 gtid, kmp_int32 *p_last,
                            kmp_int32 *p_lb, kmp_int32 *p_ub, kmp_int32 *p_st) {
-#if OMPT_SUPPORT && OMPT_OPTIONAL
-  OMPT_STORE_RETURN_ADDRESS(gtid);
-#endif
   return __kmp_dispatch_next<kmp_int32>(loc, gtid, p_last, p_lb, p_ub, p_st
 #if OMPT_SUPPORT && OMPT_OPTIONAL
                                         ,
-                                        OMPT_LOAD_RETURN_ADDRESS(gtid)
+                                        OMPT_GET_RETURN_ADDRESS(0)
 #endif
-                                            );
 }
 
 /*!
 See @ref __kmpc_dispatch_next_4
 */
 int __kmpc_dispatch_next_4u(ident_t *loc, kmp_int32 gtid, kmp_int32 *p_last,
                             kmp_uint32 *p_lb, kmp_uint32 *p_ub,
                             kmp_int32 *p_st) {
-#if OMPT_SUPPORT && OMPT_OPTIONAL
-  OMPT_STORE_RETURN_ADDRESS(gtid);
-#endif
   return __kmp_dispatch_next<kmp_uint32>(loc, gtid, p_last, p_lb, p_ub, p_st
 #if OMPT_SUPPORT && OMPT_OPTIONAL
                                          ,
-                                         OMPT_LOAD_RETURN_ADDRESS(gtid)
+                                         OMPT_GET_RETURN_ADDRESS(0)
 #endif
-                                             );
 }
 
 /*!
 See @ref __kmpc_dispatch_next_4
 */
 int __kmpc_dispatch_next_8(ident_t *loc, kmp_int32 gtid, kmp_int32 *p_last,
                            kmp_int64 *p_lb, kmp_int64 *p_ub, kmp_int64 *p_st) {
-#if OMPT_SUPPORT && OMPT_OPTIONAL
-  OMPT_STORE_RETURN_ADDRESS(gtid);
-#endif
   return __kmp_dispatch_next<kmp_int64>(loc, gtid, p_last, p_lb, p_ub, p_st
 #if OMPT_SUPPORT && OMPT_OPTIONAL
                                         ,
-                                        OMPT_LOAD_RETURN_ADDRESS(gtid)
+                                        OMPT_GET_RETURN_ADDRESS(0)
 #endif
-                                            );
 }
 
 /*!
 See @ref __kmpc_dispatch_next_4
 */
 int __kmpc_dispatch_next_8u(ident_t *loc, kmp_int32 gtid, kmp_int32 *p_last,
                             kmp_uint64 *p_lb, kmp_uint64 *p_ub,
                             kmp_int64 *p_st) {
-#if OMPT_SUPPORT && OMPT_OPTIONAL
-  OMPT_STORE_RETURN_ADDRESS(gtid);
-#endif
   return __kmp_dispatch_next<kmp_uint64>(loc, gtid, p_last, p_lb, p_ub, p_st
 #if OMPT_SUPPORT && OMPT_OPTIONAL
                                          ,
-                                         OMPT_LOAD_RETURN_ADDRESS(gtid)
+                                         OMPT_GET_RETURN_ADDRESS(0)
 #endif
-                                             );
 }
 
 /*!
 @param loc Source code location
 @param gtid Global thread id
 
 Mark the end of a dynamic loop.
 */

openmp/runtime/src/kmp_gsupport.cpp

#if OMPT_SUPPORT && OMPT_OPTIONAL
   }
   OMPT_STORE_RETURN_ADDRESS(gtid);
 #endif
   __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
 
   // Retrieve the value of the copyprivate data point, and wait for all
   // threads to do likewise, then return.
   retval = __kmp_team_from_gtid(gtid)->t.t_copypriv_data;
-  {
-#if OMPT_SUPPORT && OMPT_OPTIONAL
-    OMPT_STORE_RETURN_ADDRESS(gtid);
-#endif
-    __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
-  }
 #if OMPT_SUPPORT && OMPT_OPTIONAL
   if (ompt_enabled.enabled) {
     ompt_frame->enter_frame = ompt_data_none;
   }
 #endif
   return retval;
 }
 
#if OMPT_SUPPORT && OMPT_OPTIONAL
   ompt_frame_t *ompt_frame;
   if (ompt_enabled.enabled) {
     __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
     ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
   }
   OMPT_STORE_RETURN_ADDRESS(gtid);
 #endif
   __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
-  {
-#if OMPT_SUPPORT && OMPT_OPTIONAL
-    OMPT_STORE_RETURN_ADDRESS(gtid);
-#endif
-    __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
-  }
 #if OMPT_SUPPORT && OMPT_OPTIONAL
   if (ompt_enabled.enabled) {
     ompt_frame->enter_frame = ompt_data_none;
   }
 #endif
 }
 
 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_ORDERED_START)(void) {
#endif /* KMP_DEBUG */
     __kmp_GOMP_parallel_microtask_wrapper(int *gtid, int *npr,
                                           void (*task)(void *), void *data,
                                           unsigned num_threads, ident_t *loc,
                                           enum sched_type schedule, long start,
                                           long end, long incr,
                                           long chunk_size) {
   // Initialize the loop worksharing construct.
 
+#if OMPT_SUPPORT
+  OMPT_STORE_RETURN_ADDRESS(*gtid);
+#endif
   KMP_DISPATCH_INIT(loc, *gtid, schedule, start, end, incr, chunk_size,
                     schedule != kmp_sch_static);
 
 #if OMPT_SUPPORT
   kmp_info_t *thr;
   ompt_frame_t *ompt_frame;
   ompt_state_t enclosing_state;
 
int func(long lb, long ub, long str, long chunk_sz, long *p_lb,              \
     MKLOC(loc, KMP_STR(func));                                                 \
     KA_TRACE(                                                                  \
         20,                                                                    \
         (KMP_STR(                                                              \
              func) ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz 0x%lx\n",  \
          gtid, lb, ub, str, chunk_sz));                                        \
                                                                                \
     if ((str > 0) ? (lb < ub) : (lb > ub)) {                                   \
-      {                                                                        \
         IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);)                      \
         KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb,                          \
                           (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz,      \
                           (schedule) != kmp_sch_static);                       \
-      }                                                                        \
-      {                                                                        \
-        IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);)                      \
         status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, (kmp_int *)p_lb,          \
                                    (kmp_int *)p_ub, (kmp_int *)&stride);       \
-      }                                                                        \
       if (status) {                                                            \
         KMP_DEBUG_ASSERT(stride == str);                                       \
         *p_ub += (str > 0) ? 1 : -1;                                           \
       }                                                                        \
     } else {                                                                   \
       status = 0;                                                              \
     }                                                                          \
                                                                                \
int func(long lb, long ub, long str, long *p_lb, long *p_ub) {               \
     int gtid = __kmp_entry_gtid();                                             \
     MKLOC(loc, KMP_STR(func));                                                 \
     KA_TRACE(                                                                  \
         20,                                                                    \
         (KMP_STR(func) ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz %d\n", \
          gtid, lb, ub, str, chunk_sz));                                        \
                                                                                \
     if ((str > 0) ? (lb < ub) : (lb > ub)) {                                   \
-      {                                                                        \
         IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);)                      \
         KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb,                          \
                           (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz,      \
                           TRUE);                                               \
-      }                                                                        \
-      {                                                                        \
-        IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);)                      \
         status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, (kmp_int *)p_lb,          \
                                    (kmp_int *)p_ub, (kmp_int *)&stride);       \
-      }                                                                        \
       if (status) {                                                            \
         KMP_DEBUG_ASSERT(stride == str);                                       \
         *p_ub += (str > 0) ? 1 : -1;                                           \
       }                                                                        \
     } else {                                                                   \
       status = 0;                                                              \
     }                                                                          \
                                                                                \
bool func(unsigned ncounts, long *counts, long chunk_sz, long *p_lb,         \
     ub = counts[0];                                                            \
     str = 1;                                                                   \
     KA_TRACE(20, (KMP_STR(func) ": T#%d, ncounts %u, lb 0x%lx, ub 0x%lx, str " \
                                 "0x%lx, chunk_sz "                             \
                                 "0x%lx\n",                                     \
                   gtid, ncounts, lb, ub, str, chunk_sz));                      \
                                                                                \
     if ((str > 0) ? (lb < ub) : (lb > ub)) {                                   \
+      IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);)                        \
       KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb,                            \
                         (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz,        \
                         (schedule) != kmp_sch_static);                         \
       status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, (kmp_int *)p_lb,            \
                                  (kmp_int *)p_ub, (kmp_int *)&stride);         \
       if (status) {                                                            \
         KMP_DEBUG_ASSERT(stride == str);                                       \
         *p_ub += (str > 0) ? 1 : -1;                                           \
int func(unsigned ncounts, long *counts, long *p_lb, long *p_ub) {           \
     ub = counts[0];                                                            \
     str = 1;                                                                   \
     KA_TRACE(                                                                  \
         20,                                                                    \
         (KMP_STR(func) ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz %d\n", \
          gtid, lb, ub, str, chunk_sz));                                        \
                                                                                \
     if ((str > 0) ? (lb < ub) : (lb > ub)) {                                   \
+      IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);)                        \
       KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb,                            \
                         (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, TRUE); \
       status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, (kmp_int *)p_lb,            \
                                  (kmp_int *)p_ub, (kmp_int *)&stride);         \
       if (status) {                                                            \
         KMP_DEBUG_ASSERT(stride == str);                                       \
         *p_ub += (str > 0) ? 1 : -1;                                           \
       }                                                                        \
void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_END)(void) {
   int gtid = __kmp_get_gtid();
   KA_TRACE(20, ("GOMP_loop_end: T#%d\n", gtid))
 
 #if OMPT_SUPPORT && OMPT_OPTIONAL
   ompt_frame_t *ompt_frame;
   if (ompt_enabled.enabled) {
     __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
     ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
-    OMPT_STORE_RETURN_ADDRESS(gtid);
   }
+  OMPT_STORE_RETURN_ADDRESS(gtid);
 #endif
   __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
 #if OMPT_SUPPORT && OMPT_OPTIONAL
   if (ompt_enabled.enabled) {
     ompt_frame->enter_frame = ompt_data_none;
   }
 #endif
 
void func(void (*task)(void *), void *data, unsigned num_threads, long lb,   \
     KA_TRACE(                                                                  \
         20,                                                                    \
         (KMP_STR(                                                              \
              func) ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz 0x%lx\n",  \
          gtid, lb, ub, str, chunk_sz));                                        \
                                                                                \
     ompt_pre();                                                                \
                                                                                \
+    IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);)                          \
     __kmp_GOMP_fork_call(&loc, gtid, num_threads, 0u, task,                    \
                          (microtask_t)__kmp_GOMP_parallel_microtask_wrapper,   \
                          9, task, data, num_threads, &loc, (schedule), lb,     \
                          (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz);      \
-    IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid));                          \
                                                                                \
     KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb,                              \
                       (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz,          \
                       (schedule) != kmp_sch_static);                           \
                                                                                \
     ompt_post();                                                               \
                                                                                \
     KA_TRACE(20, (KMP_STR(func) " exit: T#%d\n", gtid));                       \
   }
 
 #if OMPT_SUPPORT && OMPT_OPTIONAL
 
 #define OMPT_LOOP_PRE()                                                        \
   ompt_frame_t *parent_frame;                                                  \
   if (ompt_enabled.enabled) {                                                  \
     __ompt_get_task_info_internal(0, NULL, NULL, &parent_frame, NULL, NULL);   \
     parent_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);                 \
-    OMPT_STORE_RETURN_ADDRESS(gtid);                                           \
   }
 
 #define OMPT_LOOP_POST()                                                       \
   if (ompt_enabled.enabled) {                                                  \
     parent_frame->enter_frame = ompt_data_none;                                \
   }
 
 #else
void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASK)(void (*func)(void *), void *data,
   }
 
 #if OMPT_SUPPORT
   kmp_taskdata_t *current_task;
   if (ompt_enabled.enabled) {
     current_task = __kmp_threads[gtid]->th.th_current_task;
     current_task->ompt_task_info.frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
   }
-  OMPT_STORE_RETURN_ADDRESS(gtid);
 #endif
 
   if (if_cond) {
+#if OMPT_SUPPORT
+    OMPT_STORE_RETURN_ADDRESS(gtid);
+#endif
     if (gomp_flags & KMP_GOMP_TASK_DEPENDS_FLAG) {
       KMP_ASSERT(depend);
       kmp_gomp_depends_info_t gomp_depends(depend);
       kmp_int32 ndeps = gomp_depends.get_num_deps();
       kmp_depend_info_t dep_list[ndeps];
       for (kmp_int32 i = 0; i < ndeps; i++)
         dep_list[i] = gomp_depends.get_kmp_depend(i);
       __kmp_aux_omp_task_with_deps(&loc, gtid, task, ndeps, dep_list, 0, NULL);
     } else {
       __kmp_aux_omp_task(&loc, gtid, task);
     }
   } else {
 #if OMPT_SUPPORT
     ompt_thread_info_t oldInfo;
     kmp_info_t *thread;
     kmp_taskdata_t *taskdata;
     if (ompt_enabled.enabled) {
       // Store the threads states and restore them after the task
       thread = __kmp_threads[gtid];
       taskdata = KMP_TASK_TO_TASKDATA(task);
       oldInfo = thread->th.ompt_thread_info;
       thread->th.ompt_thread_info.wait_id = 0;
       thread->th.ompt_thread_info.state = ompt_state_work_parallel;
-      taskdata->ompt_task_info.frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
+      //      taskdata->ompt_task_info.frame.exit_frame.ptr =
+      //      OMPT_GET_FRAME_ADDRESS(0);
     }
-    OMPT_STORE_RETURN_ADDRESS(gtid);
 #endif
     if (gomp_flags & KMP_GOMP_TASK_DEPENDS_FLAG) {
       KMP_ASSERT(depend);
       kmp_gomp_depends_info_t gomp_depends(depend);
       kmp_int32 ndeps = gomp_depends.get_num_deps();
       kmp_depend_info_t dep_list[ndeps];
       for (kmp_int32 i = 0; i < ndeps; i++)
         dep_list[i] = gomp_depends.get_kmp_depend(i);
+#if OMPT_SUPPORT
+      OMPT_STORE_RETURN_ADDRESS(gtid);
+#endif
       __kmp_aux_omp_wait_deps(&loc, gtid, ndeps, dep_list, 0, NULL);
     }
 
 #if OMPT_SUPPORT
     if (UNLIKELY(ompt_enabled.enabled)) {
       __kmp_omp_task_begin_if0_ompt(&loc, gtid, task, OMPT_GET_FRAME_ADDRESS(1),
                                     OMPT_GET_FRAME_ADDRESS(0),
                                     OMPT_GET_RETURN_ADDRESS(0));
#endif
 
   KA_TRACE(20, ("GOMP_task exit: T#%d\n", gtid));
 }
 
 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKWAIT)(void) {
   MKLOC(loc, "GOMP_taskwait");
   int gtid = __kmp_entry_gtid();
 
-#if OMPT_SUPPORT
-  OMPT_STORE_RETURN_ADDRESS(gtid);
-#endif
-
   KA_TRACE(20, ("GOMP_taskwait: T#%d\n", gtid));
 
 #if OMPT_SUPPORT && OMPT_OPTIONAL
   if (UNLIKELY(ompt_enabled.enabled)) {
     __kmp_omp_taskwait_ompt(&loc, gtid, OMPT_GET_FRAME_ADDRESS(0),
                             OMPT_GET_RETURN_ADDRESS(0));
   } else
 #endif
__kmp_GOMP_fork_call(&loc, gtid, num_threads, flags, task,
                        data);
 #if OMPT_SUPPORT
   if (ompt_enabled.enabled) {
     task_info = __ompt_get_task_info_object(0);
     task_info->frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
   }
 #endif
   task(data);
-  {
-#if OMPT_SUPPORT
-    OMPT_STORE_RETURN_ADDRESS(gtid);
-#endif
-    KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_END)();
-  }
 #if OMPT_SUPPORT
   if (ompt_enabled.enabled) {
     task_info->frame.exit_frame = ompt_data_none;
     parent_task_info->frame.enter_frame = ompt_data_none;
   }
 #endif
 }
 
#if OMPT_SUPPORT
   OMPT_STORE_RETURN_ADDRESS(gtid);
 #endif
 
   __kmp_GOMP_fork_call(&loc, gtid, num_threads, flags, task,
                        (microtask_t)__kmp_GOMP_parallel_microtask_wrapper, 9,
                        task, data, num_threads, &loc, kmp_nm_dynamic_chunked,
                        (kmp_int)1, (kmp_int)count, (kmp_int)1, (kmp_int)1);
 
-  {
-#if OMPT_SUPPORT
-  OMPT_STORE_RETURN_ADDRESS(gtid);
-#endif
-
   KMP_DISPATCH_INIT(&loc, gtid, kmp_nm_dynamic_chunked, 1, count, 1, 1, TRUE);
-  }
   task(data);
   KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_END)();
   KA_TRACE(20, ("GOMP_parallel_sections exit: T#%d\n", gtid));
 }
 
 #define PARALLEL_LOOP(func, schedule, ompt_pre, ompt_post)                     \
   void func(void (*task)(void *), void *data, unsigned num_threads, long lb,   \
             long ub, long str, long chunk_sz, unsigned flags) {                \
     int gtid = __kmp_entry_gtid();                                             \
     MKLOC(loc, KMP_STR(func));                                                 \
     KA_TRACE(                                                                  \
         20,                                                                    \
         (KMP_STR(                                                              \
              func) ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz 0x%lx\n",  \
          gtid, lb, ub, str, chunk_sz));                                        \
                                                                                \
     ompt_pre();                                                                \
     IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);)                          \
     __kmp_GOMP_fork_call(&loc, gtid, num_threads, flags, task,                 \
                          (microtask_t)__kmp_GOMP_parallel_microtask_wrapper,   \
                          9, task, data, num_threads, &loc, (schedule), lb,     \
                          (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz);      \
                                                                                \
-    {                                                                          \
-      IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);)                        \
-      KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb,                            \
-                        (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz,        \
-                        (schedule) != kmp_sch_static);                         \
-    }                                                                          \
+    KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb,                              \
+                      (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz,          \
+                      (schedule) != kmp_sch_static);                           \
     task(data);                                                                \
     KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_END)();                         \
     ompt_post();                                                               \
                                                                                \
     KA_TRACE(20, (KMP_STR(func) " exit: T#%d\n", gtid));                       \
   }
 
 PARALLEL_LOOP(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_STATIC),

 }
 
 // Return true if cancellation should take place, false otherwise
 bool KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CANCELLATION_POINT)(int which) {
   int gtid = __kmp_get_gtid();
   MKLOC(loc, "GOMP_cancellation_point");
   KA_TRACE(20, ("GOMP_cancellation_point: T#%d which:%d\n", gtid, which));
   kmp_int32 cncl_kind = __kmp_gomp_to_omp_cancellation_kind(which);
+#if OMPT_SUPPORT
+  OMPT_STORE_RETURN_ADDRESS(gtid);
+#endif
   return __kmp_aux_cancellationpoint(&loc, gtid, cncl_kind);
 }
 
 // Return true if cancellation should take place, false otherwise
 bool KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CANCEL)(int which, bool do_cancel) {
   int gtid = __kmp_get_gtid();
   MKLOC(loc, "GOMP_cancel");
   KA_TRACE(20, ("GOMP_cancel: T#%d which:%d do_cancel:%d\n", gtid, which,
                 (int)do_cancel));
   kmp_int32 cncl_kind = __kmp_gomp_to_omp_cancellation_kind(which);
+#if OMPT_SUPPORT
+  OMPT_STORE_RETURN_ADDRESS(gtid);
+#endif
 
   if (do_cancel == FALSE) {
     return __kmp_aux_cancellationpoint(&loc, gtid, cncl_kind);
   } else {
     return __kmp_aux_cancel(&loc, gtid, cncl_kind);
   }
 }
 

 #ifdef __cplusplus
 extern "C" {
 #endif // __cplusplus
 
 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKLOOP)(
     void (*func)(void *), void *data, void (*copy_func)(void *, void *),
     long arg_size, long arg_align, unsigned gomp_flags, unsigned long num_tasks,
     int priority, long start, long end, long step) {
+#if OMPT_SUPPORT
+  int gtid = __kmp_entry_gtid();
+  OMPT_STORE_RETURN_ADDRESS(gtid);
+#endif
   __GOMP_taskloop<long>(func, data, copy_func, arg_size, arg_align, gomp_flags,
                         num_tasks, priority, start, end, step);
 }
 
 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKLOOP_ULL)(
     void (*func)(void *), void *data, void (*copy_func)(void *, void *),
     long arg_size, long arg_align, unsigned gomp_flags, unsigned long num_tasks,
     int priority, unsigned long long start, unsigned long long end,

openmp/runtime/src/kmp_runtime.cpp

if (this_thr->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {
     proc_bind = this_thr->th.th_current_task->td_icvs.proc_bind;
   }
   // Reset for next parallel region
   this_thr->th.th_set_proc_bind = proc_bind_default;
 
 #if OMPT_SUPPORT
   ompt_data_t ompt_parallel_data = ompt_data_none;
   ompt_data_t *implicit_task_data;
-  void *codeptr = OMPT_LOAD_RETURN_ADDRESS(global_tid);
+  void *codeptr;
   if (ompt_enabled.enabled &&
       this_thr->th.ompt_thread_info.state != ompt_state_overhead) {
-
+    codeptr = OMPT_LOAD_RETURN_ADDRESS(global_tid);
     ompt_task_info_t *parent_task_info;
     parent_task_info = OMPT_CUR_TASK_INFO(this_thr);
 
     parent_task_info->frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
     if (ompt_enabled.ompt_callback_parallel_begin) {
       int team_size = 1;
 
       ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)(
if (this_thr->th.th_prev_level != serial_team->t.t_level ||
       this_thr->th.th_prev_level = serial_team->t.t_level;
       this_thr->th.th_prev_num_threads = 1;
     }
   }
 
   if (__kmp_env_consistency_check)
     __kmp_push_parallel(global_tid, NULL);
 #if OMPT_SUPPORT
-  serial_team->t.ompt_team_info.master_return_address = codeptr;
   if (ompt_enabled.enabled &&
       this_thr->th.ompt_thread_info.state != ompt_state_overhead) {
     OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
 
     ompt_lw_taskteam_t lw_taskteam;
     __ompt_lw_taskteam_init(&lw_taskteam, this_thr, global_tid,
                             &ompt_parallel_data, codeptr);
 
int __kmp_fork_call(ident_t *loc, int gtid,
   int master_active;
   int master_set_numthreads;
   int level;
   int active_level;
   int teams_level;
 #if KMP_NESTED_HOT_TEAMS
   kmp_hot_team_ptr_t **p_hot_teams;
 #endif
+#if OMPT_SUPPORT
+  void *return_address = NULL;
+#endif
   { // KMP_TIME_BLOCK
     KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_fork_call);
     KMP_COUNT_VALUE(OMP_PARALLEL_args, argc);
 
     KA_TRACE(20, ("__kmp_fork_call: enter T#%d\n", gtid));
     if (__kmp_stkpadding > 0 && __kmp_root[gtid] != NULL) {
       /* Some systems prefer the stack for the root thread(s) to start with */
       /* some gap from the parent stack to prevent false sharing. */
#endif
     master_active = root->r.r_active;
     master_set_numthreads = master_th->th.th_set_nproc;
 
 #if OMPT_SUPPORT
     ompt_data_t ompt_parallel_data = ompt_data_none;
     ompt_data_t *parent_task_data;
     ompt_frame_t *ompt_frame;
     ompt_data_t *implicit_task_data;
-    void *return_address = NULL;
 
     if (ompt_enabled.enabled) {
       __ompt_get_task_info_internal(0, NULL, &parent_task_data, &ompt_frame,
                                     NULL, NULL);
-      return_address = OMPT_LOAD_RETURN_ADDRESS(gtid);
+      // for combined #pragma omp teams parallel, the non-root initial threads
+      // might not have a return address, so we will not assert here.
+      return_address = OMPT_LOAD_RETURN_ADDRESS_OR_NULL(gtid);
     }
 #endif
 
     // Nested level will be an index in the nested nthreads array
     level = parent_team->t.t_level;
     // used to launch non-serial teams even if nested is not allowed
     active_level = parent_team->t.t_active_level;
     // needed to check nesting inside the teams

openmp/runtime/src/kmp_taskdeps.cpp

if (!current_task->ompt_task_info.frame.enter_frame.ptr)
           OMPT_GET_FRAME_ADDRESS(0);
     if (ompt_enabled.ompt_callback_task_create) {
       ompt_data_t task_data = ompt_data_none;
       ompt_callbacks.ompt_callback(ompt_callback_task_create)(
           current_task ? &(current_task->ompt_task_info.task_data) : &task_data,
           current_task ? &(current_task->ompt_task_info.frame) : NULL,
           &(new_taskdata->ompt_task_info.task_data),
           ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(new_taskdata), 1,
-          OMPT_LOAD_OR_GET_RETURN_ADDRESS(gtid));
+          OMPT_LOAD_RETURN_ADDRESS(gtid));
     }
 
     new_taskdata->ompt_task_info.frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
   }
 
 #if OMPT_OPTIONAL
   /* OMPT grab all dependences if requested by the tool */
   if (ndeps + ndeps_noalias > 0 &&
#endif
   return ret;
 }
 
 kmp_int32 __kmpc_omp_task_with_deps(ident_t *loc_ref, kmp_int32 gtid,
                                     kmp_task_t *new_task, kmp_int32 ndeps,
                                     kmp_depend_info_t *dep_list,
                                     kmp_int32 ndeps_noalias,
                                     kmp_depend_info_t *noalias_dep_list) {
+#if OMPT_SUPPORT
+  OMPT_STORE_RETURN_ADDRESS(gtid);
+#endif
   return __kmp_aux_omp_task_with_deps(loc_ref, gtid, new_task, ndeps, dep_list,
                                       ndeps_noalias, noalias_dep_list);
 }
 
 #if OMPT_SUPPORT
 void __ompt_taskwait_dep_finish(kmp_taskdata_t *current_task,
                                 ompt_data_t *taskwait_task_data) {
   if (ompt_enabled.ompt_callback_task_schedule) {
if (!current_task->ompt_task_info.frame.enter_frame.ptr)
           OMPT_GET_FRAME_ADDRESS(0);
     if (ompt_enabled.ompt_callback_task_create) {
       ompt_data_t task_data = ompt_data_none;
       ompt_callbacks.ompt_callback(ompt_callback_task_create)(
           current_task ? &(current_task->ompt_task_info.task_data) : &task_data,
           current_task ? &(current_task->ompt_task_info.frame) : NULL,
           taskwait_task_data,
           ompt_task_explicit | ompt_task_undeferred | ompt_task_mergeable, 1,
-          OMPT_LOAD_OR_GET_RETURN_ADDRESS(gtid));
+          OMPT_LOAD_RETURN_ADDRESS(gtid));
     }
   }
 
 #if OMPT_OPTIONAL
   /* OMPT grab all dependences if requested by the tool */
   if (ndeps + ndeps_noalias > 0 && ompt_enabled.ompt_callback_dependences) {
     kmp_int32 i;
 

 #endif /* OMPT_SUPPORT */
   KA_TRACE(10, ("__kmpc_omp_wait_deps(exit): T#%d finished waiting : loc=%p\n",
                 gtid, loc_ref));
 }
 
 void __kmpc_omp_wait_deps(ident_t *loc_ref, kmp_int32 gtid, kmp_int32 ndeps,
                           kmp_depend_info_t *dep_list, kmp_int32 ndeps_noalias,
                           kmp_depend_info_t *noalias_dep_list) {
+#if OMPT_SUPPORT
+  OMPT_STORE_RETURN_ADDRESS(gtid);
+#endif
   return __kmp_aux_omp_wait_deps(loc_ref, gtid, ndeps, dep_list, ndeps_noalias,
                                  noalias_dep_list);
 }

openmp/runtime/src/kmp_tasking.cpp

 #endif
   KA_TRACE(10, ("__kmpc_omp_task(enter): T#%d loc=%p task=%p\n", gtid, loc_ref,
                 new_taskdata));
   __kmp_assert_valid_gtid(gtid);
 
 #if OMPT_SUPPORT
   kmp_taskdata_t *parent = NULL;
   if (UNLIKELY(ompt_enabled.enabled)) {
+    void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
     if (!new_taskdata->td_flags.started) {
-      OMPT_STORE_RETURN_ADDRESS(gtid);
       parent = new_taskdata->td_parent;
       if (!parent->ompt_task_info.frame.enter_frame.ptr) {
         parent->ompt_task_info.frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
       }
       if (ompt_enabled.ompt_callback_task_create) {
         ompt_data_t task_data = ompt_data_none;
         ompt_callbacks.ompt_callback(ompt_callback_task_create)(
             parent ? &(parent->ompt_task_info.task_data) : &task_data,
             parent ? &(parent->ompt_task_info.frame) : NULL,
             &(new_taskdata->ompt_task_info.task_data),
             ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(new_taskdata), 0,
-            OMPT_LOAD_RETURN_ADDRESS(gtid));
+            codeptr);
       }
     } else {
       // We are scheduling the continuation of an UNTIED task.
       // Scheduling back to the parent task.
       __ompt_task_finish(new_task,
                          new_taskdata->ompt_task_info.scheduling_parent,
                          ompt_task_switch);
       new_taskdata->ompt_task_info.frame.exit_frame = ompt_data_none;
#if OMPT_SUPPORT
   if (UNLIKELY(ompt_enabled.enabled && parent != NULL)) {
     parent->ompt_task_info.frame.enter_frame = ompt_data_none;
   }
 #endif
   return res;
 }
 kmp_int32 __kmpc_omp_task(ident_t *loc_ref, kmp_int32 gtid,
                           kmp_task_t *new_task) {
+#if OMPT_SUPPORT && OMPT_OPTIONAL
+  OMPT_STORE_RETURN_ADDRESS(gtid);
+#endif
   return __kmp_aux_omp_task(loc_ref, gtid, new_task);
 }
 
 // __kmp_omp_taskloop_task: Wrapper around __kmp_omp_task to schedule
 // a taskloop task with the correct OMPT return address
 //
 // loc_ref: location of original task pragma (ignored)
 // gtid: Global Thread ID of encountering thread

 }
 #endif // OMPT_SUPPORT && OMPT_OPTIONAL
 
 // __kmpc_omp_taskwait: Wait until all tasks generated by the current task are
 // complete
 kmp_int32 __kmpc_omp_taskwait(ident_t *loc_ref, kmp_int32 gtid) {
 #if OMPT_SUPPORT && OMPT_OPTIONAL
   if (UNLIKELY(ompt_enabled.enabled)) {
-    OMPT_STORE_RETURN_ADDRESS(gtid);
     return __kmp_omp_taskwait_ompt(loc_ref, gtid, OMPT_GET_FRAME_ADDRESS(0),
                                    OMPT_GET_RETURN_ADDRESS(0));
   }
 #endif
   return __kmp_omp_taskwait_template<false>(loc_ref, gtid, NULL, NULL);
 }
 
 // __kmpc_omp_taskyield: switch to a different task
void __kmp_aux_taskgroup(ident_t *loc, int gtid) {
   tg_new->parent = taskdata->td_taskgroup;
   tg_new->reduce_data = NULL;
   tg_new->reduce_num_data = 0;
   taskdata->td_taskgroup = tg_new;
 
 #if OMPT_SUPPORT && OMPT_OPTIONAL
   if (UNLIKELY(ompt_enabled.ompt_callback_sync_region)) {
     void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
-    if (!codeptr)
-      codeptr = OMPT_GET_RETURN_ADDRESS(0);
     kmp_team_t *team = thread->th.th_team;
     ompt_data_t my_task_data = taskdata->ompt_task_info.task_data;
     // FIXME: I think this is wrong for lwt!
     ompt_data_t my_parallel_data = team->t.ompt_team_info.parallel_data;
 
     ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
         ompt_sync_region_taskgroup, ompt_scope_begin, &(my_parallel_data),
         &(my_task_data), codeptr);
   }
 #endif
 }
 void __kmpc_taskgroup(ident_t *loc, int gtid) {
+#if OMPT_SUPPORT && OMPT_OPTIONAL
+  OMPT_STORE_RETURN_ADDRESS(gtid);
+#endif
   __kmp_aux_taskgroup(loc, gtid);
 }
 
 // __kmpc_end_taskgroup: Wait until all tasks generated by the current task
 //                       and its descendants are complete
 void __kmp_aux_end_taskgroup(ident_t *loc, int gtid) {
   __kmp_assert_valid_gtid(gtid);
   kmp_info_t *thread = __kmp_threads[gtid];
   kmp_taskdata_t *taskdata = thread->th.th_current_task;
   kmp_taskgroup_t *taskgroup = taskdata->td_taskgroup;
   int thread_finished = FALSE;
 
 #if OMPT_SUPPORT && OMPT_OPTIONAL
   kmp_team_t *team;
   ompt_data_t my_task_data;
   ompt_data_t my_parallel_data;
   void *codeptr;
   if (UNLIKELY(ompt_enabled.enabled)) {
     team = thread->th.th_team;
     my_task_data = taskdata->ompt_task_info.task_data;
     // FIXME: I think this is wrong for lwt!
     my_parallel_data = team->t.ompt_team_info.parallel_data;
     codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
-    if (!codeptr)
-      codeptr = OMPT_GET_RETURN_ADDRESS(0);
   }
 #endif
 
   KA_TRACE(10, ("__kmpc_end_taskgroup(enter): T#%d loc=%p\n", gtid, loc));
   KMP_DEBUG_ASSERT(taskgroup != NULL);
   KMP_SET_THREAD_STATE_BLOCK(TASKGROUP);
 
   if (__kmp_tasking_mode != tskm_immediate_exec) {
#if OMPT_SUPPORT && OMPT_OPTIONAL
   if (UNLIKELY(ompt_enabled.ompt_callback_sync_region)) {
     ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
         ompt_sync_region_taskgroup, ompt_scope_end, &(my_parallel_data),
         &(my_task_data), codeptr);
   }
 #endif
 }
 void __kmpc_end_taskgroup(ident_t *loc, int gtid) {
+#if OMPT_SUPPORT && OMPT_OPTIONAL
+  OMPT_STORE_RETURN_ADDRESS(gtid);
+#endif
   __kmp_aux_end_taskgroup(loc, gtid);
 }
 
 // __kmp_remove_my_task: remove a task from my own deque
 static kmp_task_t *__kmp_remove_my_task(kmp_info_t *thread, kmp_int32 gtid,
                                         kmp_task_team_t *task_team,
                                         kmp_int32 is_constrained) {
   kmp_task_t *task;

 @param grainsize Schedule value if specified
 @param task_dup  Tasks duplication routine
 
 Execute the taskloop construct.
 */
 void __kmp_aux_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
                         kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
                         int nogroup, int sched, kmp_uint64 grainsize,
+#if OMPT_SUPPORT
+                        void *codeptr_ra,
+#endif
                         void *task_dup) {
   kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
   KMP_DEBUG_ASSERT(task != NULL);
   __kmp_assert_valid_gtid(gtid);
   if (nogroup == 0) {
-#if OMPT_SUPPORT && OMPT_OPTIONAL
-    OMPT_STORE_RETURN_ADDRESS(gtid);
-#endif
     __kmp_aux_taskgroup(loc, gtid);
   }
 
   // =========================================================================
   // calculate loop parameters
   kmp_taskloop_bounds_t task_bounds(task, lb, ub);
   kmp_uint64 tc;
   // compiler provides global bounds here
#endif
   }
 
 #if OMPT_SUPPORT && OMPT_OPTIONAL
   ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL);
   ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
   if (ompt_enabled.ompt_callback_work) {
     ompt_callbacks.ompt_callback(ompt_callback_work)(
         ompt_work_taskloop, ompt_scope_begin, &(team_info->parallel_data),
-        &(task_info->task_data), tc, OMPT_GET_RETURN_ADDRESS(0));
+        &(task_info->task_data), tc, codeptr_ra);
   }
 #endif
 
   if (num_tasks_min == 0)
     // TODO: can we choose better default heuristic?
     num_tasks_min =
         KMP_MIN(thread->th.th_team_nproc * 10, INITIAL_TASK_DEQUE_SIZE);
 
#endif
   // Also require GOMP_taskloop to reduce to linear (taskdata->td_flags.native)
   if (if_val == 0) { // if(0) specified, mark task as serial
     taskdata->td_flags.task_serial = 1;
     taskdata->td_flags.tiedness = TASK_TIED; // AC: serial task cannot be untied
     // always start serial tasks linearly
     __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, num_tasks,
                           grainsize, extras, tc,
 #if OMPT_SUPPORT
-                          OMPT_GET_RETURN_ADDRESS(0),
+                          codeptr_ra,
 #endif
                           task_dup);
     // !taskdata->td_flags.native => currently force linear spawning of tasks
     // for GOMP_taskloop
   } else if (num_tasks > num_tasks_min && !taskdata->td_flags.native) {
     KA_TRACE(20, ("__kmpc_taskloop: T#%d, go recursive: tc %llu, #tasks %llu"
                   "(%lld), grain %llu, extras %llu\n",
                   gtid, tc, num_tasks, num_tasks_min, grainsize, extras));
     __kmp_taskloop_recur(loc, gtid, task, lb, ub, st, ub_glob, num_tasks,
                          grainsize, extras, tc, num_tasks_min,
 #if OMPT_SUPPORT
-                         OMPT_GET_RETURN_ADDRESS(0),
+                         codeptr_ra,
 #endif
                          task_dup);
   } else {
     KA_TRACE(20, ("__kmpc_taskloop: T#%d, go linear: tc %llu, #tasks %llu"
                   "(%lld), grain %llu, extras %llu\n",
                   gtid, tc, num_tasks, num_tasks_min, grainsize, extras));
     __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, num_tasks,
                           grainsize, extras, tc,
 #if OMPT_SUPPORT
-                          OMPT_GET_RETURN_ADDRESS(0),
+                          codeptr_ra,
 #endif
                           task_dup);
   }
 
 #if OMPT_SUPPORT && OMPT_OPTIONAL
   if (ompt_enabled.ompt_callback_work) {
     ompt_callbacks.ompt_callback(ompt_callback_work)(
         ompt_work_taskloop, ompt_scope_end, &(team_info->parallel_data),
-        &(task_info->task_data), tc, OMPT_GET_RETURN_ADDRESS(0));
+        &(task_info->task_data), tc, codeptr_ra);
   }
 #endif
 
   if (nogroup == 0) {
-#if OMPT_SUPPORT && OMPT_OPTIONAL
-    OMPT_STORE_RETURN_ADDRESS(gtid);
-#endif
     __kmp_aux_end_taskgroup(loc, gtid);
   }
   KA_TRACE(20, ("__kmpc_taskloop(exit): T#%d\n", gtid));
 }
+void __kmp_aux_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
+                        kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
+                        int nogroup, int sched, kmp_uint64 grainsize,
+                        void *task_dup) {
+  __kmp_aux_taskloop(loc, gtid, task, if_val, lb, ub, st, nogroup, sched,
+                     grainsize,
+#if OMPT_SUPPORT
+                     OMPT_LOAD_RETURN_ADDRESS(gtid),
+#endif
+                     task_dup);
+}
 void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
                      kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup,
                      int sched, kmp_uint64 grainsize, void *task_dup) {
+#if OMPT_SUPPORT && OMPT_OPTIONAL
+  OMPT_STORE_RETURN_ADDRESS(gtid);
+#endif
   __kmp_aux_taskloop(loc, gtid, task, if_val, lb, ub, st, nogroup, sched,
-                     grainsize, task_dup);
+                     grainsize,
+#if OMPT_SUPPORT
+                     OMPT_GET_RETURN_ADDRESS(0),
+#endif
+                     task_dup);
 }

openmp/runtime/src/ompt-general.cpp

enum tool_setting_e {
   omp_tool_disabled,
   omp_tool_enabled
 };
 
 /*****************************************************************************
  * global variables
  ****************************************************************************/
 
-ompt_callbacks_active_t ompt_enabled;
+ompt_callbacks_active_t ompt_enabled{0};
 
 ompt_state_info_t ompt_state_info[] = {
 #define ompt_state_macro(state, code) {#state, state},
     FOREACH_OMPT_STATE(ompt_state_macro)
 #undef ompt_state_macro
 };
 
 kmp_mutex_impl_info_t kmp_mutex_impl_info[] = {
if (ompt_env_verbose_init && strcmp(ompt_env_verbose_init, "") &&
     verbose_init = 0;
 
 #if OMPT_DEBUG
   printf("ompt_pre_init(): tool_setting = %d\n", tool_setting);
 #endif
   switch (tool_setting) {
   case omp_tool_disabled:
     OMPT_VERBOSE_INIT_PRINT("OMP tool disabled. \n");
+    ompt_enabled.enabled = 0;
     break;
 
   case omp_tool_unset:
   case omp_tool_enabled:
 
     //--------------------------------------------------
     // Load tool iff specified in environment variable
     //--------------------------------------------------
     ompt_start_tool_result =
         ompt_try_start_tool(__kmp_openmp_version, ompt_get_runtime_version());
 
     memset(&ompt_enabled, 0, sizeof(ompt_enabled));
     break;
 
   case omp_tool_error:
     fprintf(stderr, "Warning: OMP_TOOL has invalid value \"%s\".\n"
                     "  legal values are (NULL,\"\",\"disabled\","
                     "\"enabled\").\n",
             ompt_env_var);
     break;
   }
   if (verbose_init && verbose_file != stderr && verbose_file != stdout)
     fclose(verbose_file);
+  ompt_enabled.initialized = 1;
 #if OMPT_DEBUG
   printf("ompt_pre_init(): ompt_enabled = %d\n", ompt_enabled);
 #endif
 }
 
 extern "C" int omp_get_initial_device(void);
 
 void ompt_post_init() {

openmp/runtime/src/ompt-internal.h

#define ompt_event_macro(event, callback, eventid)                             \
 
   FOREACH_OMPT_EVENT(ompt_event_macro)
 
 #undef ompt_event_macro
 } ompt_callbacks_internal_t;
 
 typedef struct ompt_callbacks_active_s {
   unsigned int enabled : 1;
+  unsigned int initialized : 1;
 #define ompt_event_macro(event, callback, eventid) unsigned int event : 1;
 
   FOREACH_OMPT_EVENT(ompt_event_macro)
 
 #undef ompt_event_macro
 } ompt_callbacks_active_t;
 
 #define TASK_TYPE_DETAILS_FORMAT(info)                                         \

 
 typedef struct ompt_lw_taskteam_s {
   ompt_team_info_t ompt_team_info;
   ompt_task_info_t ompt_task_info;
   int heap;
   struct ompt_lw_taskteam_s *parent;
 } ompt_lw_taskteam_t;
 
+typedef struct ompt_return_address_s {
+  struct ompt_return_address_s *next{nullptr};
+  void *addr{nullptr};
+  ompt_return_address_s(void *ra) : addr(ra) {}
+} ompt_return_address_t;
+
 typedef struct {
   ompt_data_t thread_data;
   ompt_data_t task_data; /* stored here from implicit barrier-begin until
                             implicit-task-end */
-  void *return_address; /* stored here on entry of runtime */
+  ompt_return_address_t *return_address; /* stored here on entry of runtime */
   ompt_state_t state;
   ompt_wait_id_t wait_id;
   int ompt_task_yielded;
   int parallel_flags; // information for the last parallel region invoked
   void *idle_frame;
 } ompt_thread_info_t;
 
 extern ompt_callbacks_internal_t ompt_callbacks;

openmp/runtime/src/ompt-specific.h

 #define OMPT_CUR_TEAM_INFO(thr) (&(thr->th.th_team->t.ompt_team_info))
 #define OMPT_CUR_TEAM_DATA(thr)                                                \
   (&(thr->th.th_team->t.ompt_team_info.parallel_data))
 
 #define OMPT_HAVE_WEAK_ATTRIBUTE KMP_HAVE_WEAK_ATTRIBUTE
 #define OMPT_HAVE_PSAPI KMP_HAVE_PSAPI
 #define OMPT_STR_MATCH(haystack, needle) __kmp_str_match(haystack, 0, needle)
 
-inline void *__ompt_load_return_address(int gtid) {
+template <bool assertion>
+static inline void *__ompt_load_return_address(int gtid) {
+  if (!ompt_enabled.enabled || gtid < 0)
+    return NULL;
   kmp_info_t *thr = __kmp_threads[gtid];
-  void *return_address = thr->th.ompt_thread_info.return_address;
-  thr->th.ompt_thread_info.return_address = NULL;
-  return return_address;
+  if (assertion) {
+    KMP_DEBUG_ASSERT(thr->th.ompt_thread_info.return_address != NULL &&
+                     thr->th.ompt_thread_info.return_address->addr != NULL);
+  }
+  // even if we assert above for 
+  // thr->th.ompt_thread_info.return_address == NULL, 
+  // we should still handle the case in production and avoid segfault
+  if (thr->th.ompt_thread_info.return_address == NULL) {
+    return NULL;
+  }
+  return thr->th.ompt_thread_info.return_address->addr;
 }
 
-/*#define OMPT_STORE_RETURN_ADDRESS(gtid) \
-  if (ompt_enabled.enabled && gtid >= 0 && __kmp_threads[gtid] &&              \
-      !__kmp_threads[gtid]->th.ompt_thread_info.return_address)                \
-  __kmp_threads[gtid]->th.ompt_thread_info.return_address =                    \
-      __builtin_return_address(0)*/
 #define OMPT_STORE_RETURN_ADDRESS(gtid)                                        \
   OmptReturnAddressGuard ReturnAddressGuard{gtid, __builtin_return_address(0)};
-#define OMPT_LOAD_RETURN_ADDRESS(gtid) __ompt_load_return_address(gtid)
-#define OMPT_LOAD_OR_GET_RETURN_ADDRESS(gtid)                                  \
-  ((ompt_enabled.enabled && gtid >= 0 && __kmp_threads[gtid] &&                \
-      __kmp_threads[gtid]->th.ompt_thread_info.return_address)?                \
-      __ompt_load_return_address(gtid):                                        \
-      __builtin_return_address(0))
+#define OMPT_STORE_GIVEN_RETURN_ADDRESS(gtid, addr)                            \
+  OmptReturnAddressGuard ReturnAddressGuard{gtid, addr};
+#define OMPT_LOAD_RETURN_ADDRESS(gtid) __ompt_load_return_address<true>(gtid)
+#define OMPT_LOAD_RETURN_ADDRESS_OR_NULL(gtid)                                 \
+  __ompt_load_return_address<false>(gtid)
 
 //******************************************************************************
 // inline functions
 //******************************************************************************
 
 inline kmp_info_t *ompt_get_thread_gtid(int gtid) {
   return (gtid >= 0) ? __kmp_thread_from_gtid(gtid) : NULL;
 }
 
 inline kmp_info_t *ompt_get_thread() {
   int gtid = __kmp_get_gtid();
   return ompt_get_thread_gtid(gtid);
 }
 
 inline void ompt_set_thread_state(kmp_info_t *thread, ompt_state_t state) {
   thread->th.ompt_thread_info.state = state;
 }
 
 inline const char *ompt_get_runtime_version() {
   return &__kmp_version_lib_ver[KMP_VERSION_MAGIC_LEN];
 }
 
 class OmptReturnAddressGuard {
 private:
+  ompt_return_address_t Ra;
+  ompt_return_address_t **ThreadRa;
   bool SetAddress{false};
   int Gtid;
 
 public:
-  OmptReturnAddressGuard(int Gtid, void *ReturnAddress) : Gtid(Gtid) {
-    if (ompt_enabled.enabled && Gtid >= 0 && __kmp_threads[Gtid] &&
-        !__kmp_threads[Gtid]->th.ompt_thread_info.return_address) {
+  void init(int Gtid, void *ReturnAddress) {
+    if ((ompt_enabled.enabled || !ompt_enabled.initialized) && Gtid >= 0 &&
+        __kmp_threads[Gtid]) {
+      this->Gtid = Gtid;
+      Ra.addr = ReturnAddress;
       SetAddress = true;
-      __kmp_threads[Gtid]->th.ompt_thread_info.return_address = ReturnAddress;
-    }
+      ThreadRa = &__kmp_threads[Gtid]->th.ompt_thread_info.return_address;
+      Ra.next = *ThreadRa;
+      *ThreadRa = &Ra;
+      KMP_DEBUG_ASSERT(Ra.next == NULL || Ra.addr != Ra.next->addr);
+    }
+  OmptReturnAddressGuard(int Gtid, void *ReturnAddress) : Ra(ReturnAddress) {
+    this->init(Gtid, ReturnAddress);
   }
+  OmptReturnAddressGuard() : Ra(NULL) {}
   ~OmptReturnAddressGuard() {
     if (SetAddress)
-      __kmp_threads[Gtid]->th.ompt_thread_info.return_address = NULL;
+      *ThreadRa = Ra.next;
   }
 };
 
 #endif // OMPT_SUPPORT
 
 // macros providing the OMPT callbacks for reduction clause
 #if OMPT_SUPPORT && OMPT_OPTIONAL
+#define OMPT_REDUCTION_DECL_IF(this_thr, gtid, cond)                           \
+  ompt_data_t *my_task_data;                                                   \
+  ompt_data_t *my_parallel_data;                                               \
+  void *return_address;                                                        \
+  if (cond && ompt_enabled.enabled && ompt_enabled.ompt_callback_reduction) {  \
+    my_task_data = OMPT_CUR_TASK_DATA(this_thr);                               \
+    my_parallel_data = OMPT_CUR_TEAM_DATA(this_thr);                           \
+    return_address = OMPT_LOAD_RETURN_ADDRESS(gtid);                           \
+  }
 #define OMPT_REDUCTION_DECL(this_thr, gtid)                                    \
-  ompt_data_t *my_task_data = OMPT_CUR_TASK_DATA(this_thr);                    \
-  ompt_data_t *my_parallel_data = OMPT_CUR_TEAM_DATA(this_thr);                \
-  void *return_address = OMPT_LOAD_RETURN_ADDRESS(gtid);
+  OMPT_REDUCTION_DECL_IF(this_thr, gtid, 1)
 #define OMPT_REDUCTION_BEGIN                                                   \
   if (ompt_enabled.enabled && ompt_enabled.ompt_callback_reduction) {          \
     ompt_callbacks.ompt_callback(ompt_callback_reduction)(                     \
         ompt_sync_region_reduction, ompt_scope_begin, my_parallel_data,        \
         my_task_data, return_address);                                         \
   }
 #define OMPT_REDUCTION_END                                                     \
   if (ompt_enabled.enabled && ompt_enabled.ompt_callback_reduction) {          \
     ompt_callbacks.ompt_callback(ompt_callback_reduction)(                     \
         ompt_sync_region_reduction, ompt_scope_end, my_parallel_data,          \
         my_task_data, return_address);                                         \
   }
 #else // OMPT_SUPPORT && OMPT_OPTIONAL
+#define OMPT_REDUCTION_DECL_IF(this_thr, gtid, cond)
 #define OMPT_REDUCTION_DECL(this_thr, gtid)
 #define OMPT_REDUCTION_BEGIN
 #define OMPT_REDUCTION_END
 #endif // ! OMPT_SUPPORT && OMPT_OPTIONAL
 
 #endif

openmp/runtime/test/ompt/cancel/cancel_parallel.c

 // RUN: %libomp-compile && env OMP_CANCELLATION=true %libomp-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
-// Current GOMP interface implementation does not support cancellation
-// XFAIL: gcc
 
 #include "callback.h"
 #include "omp.h"
 
 int main() {
   #pragma omp parallel num_threads(2)
   {
     if (omp_get_thread_num() == 0) {

openmp/runtime/test/ompt/cancel/cancel_taskgroup.c

-// RUN:  %libomp-compile && env OMP_CANCELLATION=true %libomp-run | %sort-threads | FileCheck %s
-// REQUIRES: ompt
-// UNSUPPORTED: clang-3, clang-4.0.0
-// Current GOMP interface implementation does not support cancellation; icc 16 has a bug
-// XFAIL: gcc, icc-16
+// RUN:  %libomp-compile && env OMP_CANCELLATION=true %libomp-run |
+// %sort-threads | FileCheck %s REQUIRES: ompt UNSUPPORTED: clang-3, clang-4.0.0
+// icc 16 has a bug:
+// UNSUPPORTED: icc-16
 
 #include "callback.h"
 #include <unistd.h>  
 #include <stdio.h>
 
 int main()
 {
   int condition=0;
-  #pragma omp parallel num_threads(2)
-  {}
+  int nthreads = omp_get_max_threads();
 
   print_frame(0);
   #pragma omp parallel num_threads(2)
   {
     #pragma omp master
     {
       #pragma omp taskgroup
       {
int main()
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_masked'
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_task_create'
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_task_schedule' 
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_cancel' 
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_thread_begin'
 
   // CHECK: {{^}}0: NULL_POINTER=[[NULL:.*$]]
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_masked_begin:
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin:
   // CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-9]+]],
-  // CHECK-SAME: task_id=[[PARENT_TASK_ID:[0-9]+]],
   // CHECK-SAME: codeptr_ra={{0x[0-f]*}}
 
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin:
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]],
+  // CHECK-SAME: task_id=[[PARENT_TASK_ID:[0-9]+]],
+
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create: parent_task_id=[[PARENT_TASK_ID]], parent_task_frame.exit={{0x[0-f]*}}, parent_task_frame.reenter={{0x[0-f]*}}, new_task_id=[[FIRST_TASK_ID:[0-9]+]], codeptr_ra={{0x[0-f]*}}, task_type=ompt_task_explicit=4, has_dependences=no
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create: parent_task_id=[[PARENT_TASK_ID]], parent_task_frame.exit={{0x[0-f]*}}, parent_task_frame.reenter={{0x[0-f]*}}, new_task_id=[[SECOND_TASK_ID:[0-9]+]], codeptr_ra={{0x[0-f]*}}, task_type=ompt_task_explicit=4, has_dependences=no
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create: parent_task_id=[[PARENT_TASK_ID]], parent_task_frame.exit={{0x[0-f]*}}, parent_task_frame.reenter={{0x[0-f]*}}, new_task_id=[[THIRD_TASK_ID:[0-9]+]], codeptr_ra={{0x[0-f]*}}, task_type=ompt_task_explicit=4, has_dependences=no
 
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create: parent_task_id=[[PARENT_TASK_ID]], parent_task_frame.exit={{0x[0-f]*}}, parent_task_frame.reenter={{0x[0-f]*}}, new_task_id=[[CANCEL_TASK_ID:[0-9]+]], codeptr_ra={{0x[0-f]*}}, task_type=ompt_task_explicit|ompt_task_undeferred=134217732, has_dependences=no
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_schedule: first_task_id=[[PARENT_TASK_ID]], second_task_id=[[CANCEL_TASK_ID]], prior_task_status=ompt_task_switch=7
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_cancel: task_data=[[CANCEL_TASK_ID]], flags=ompt_cancel_taskgroup|ompt_cancel_activated=24, codeptr_ra={{0x[0-f]*}}
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_schedule: first_task_id=[[CANCEL_TASK_ID]], second_task_id=[[PARENT_TASK_ID]], prior_task_status=ompt_task_cancel=3