[OpenMP] Add support for ompt_callback_dispatch

[hbae]

Summary

This change adds support for ompt_callback_dispatch with the new
dispatch chunk type introduced in 5.2. Definitions of the new
ompt_work_loop types were also added (but not used now).

[AndreyChurbanov]

openmp/runtime/test/ompt/teams/distribute_dispatch.c

3

Missing

// REQUIRES: ompt

openmp/runtime/test/ompt/worksharing/for/loop_dispatch.c

3

Missing

// REQUIRES: ompt

openmp/runtime/test/ompt/worksharing/sections_dispatch.c

3

Missing

// REQUIRES: ompt

[hbae]

Added missing REQUIRES line.

[AndreyChurbanov]

LGTM

[protze.joachim]

Thanks for working on this!

Just one comment inline.

openmp/runtime/test/ompt/callback.h

792–804

Can you probably add a TODO, that we should add a new attribute here: ", schedule=%s" ... ompt_schedule_values[wstype]?

This can be added with the specialization change of ompt_work_loop.

[hbae]

openmp/runtime/test/ompt/callback.h

792–804

Pushed my change after adding the suggested comments.

[tianshilei1992]

openmp/runtime/src/ompt-internal.h

60

This LoC creates tens of compilation warning. Please fix it.

In file included from /home/shiltian/Documents/vscode/llvm-project/openmp/runtime/src/kmp_affinity.cpp:13:
In file included from /home/shiltian/Documents/vscode/llvm-project/openmp/runtime/src/kmp.h:139:
/home/shiltian/Documents/vscode/llvm-project/openmp/runtime/src/ompt-internal.h:55:15: warning: anonymous non-C-compatible type given name for linkage purposes by typedef declaration; add a tag name here [-Wnon-c-typedef-for-linkage]
typedef struct {
              ^
               ompt_task_info_t
/home/shiltian/Documents/vscode/llvm-project/openmp/runtime/src/ompt-internal.h:60:42: note: type is not C-compatible due to this default member initializer
  ompt_dispatch_chunk_t dispatch_chunk = {0, 0};
                                         ^~~~~~
/home/shiltian/Documents/vscode/llvm-project/openmp/runtime/src/ompt-internal.h:61:3: note: type is given name 'ompt_task_info_t' for linkage purposes by this typedef declaration
} ompt_task_info_t;
  ^
1 warning generated.

[aganea]

Hello! It seems the loop_dispatch.c test is occasionally causing errors on one of the builders: https://lab.llvm.org/buildbot/#/builders/84/builds/22238/steps/6/logs/FAIL__libomp__loop_dispatch_c (also see the builder's history)
Anyone would have a chance to look at it please? Thanks in advance!

[hbae]

Is it possible for me to access the output of the failed test shown in the log?
/b/1/openmp-clang-x86_64-linux-debian/llvm.build/projects/openmp/runtime/test/ompt/worksharing/for/Output/loop_dispatch.c.tmp.out

[aganea]

+ @gribozavr2

[ychen]

In D122107#3439169, @aganea wrote:

Hello! It seems the loop_dispatch.c test is occasionally causing errors on one of the builders: https://lab.llvm.org/buildbot/#/builders/84/builds/22238/steps/6/logs/FAIL__libomp__loop_dispatch_c (also see the builder's history)
Anyone would have a chance to look at it please? Thanks in advance!

yep, I saw this too. https://lab.llvm.org/buildbot/#/builders/84/builds/23076

[protze.joachim]

openmp/runtime/test/ompt/worksharing/for/loop_dispatch.c

11–29

@hbae because of dynamic schedule it is possible that some thread gets no iteration. I'm not sure how the runtime behaves in such case.

I think we should replace the delay by signal-wait code similar to the tasking tests to make sure that all threads get at least one iteration.

Although nthreads should be 4 most of the time, with limited resources, the runtime might use less threads. In that case we should not deadlock - the test will fail nevertheless.

openmp/runtime/src/include/omp-tools.h.var

 typedef enum ompt_scope_endpoint_t {
   ompt_scope_begin                    = 1,
   ompt_scope_end                      = 2,
   ompt_scope_beginend                 = 3
 } ompt_scope_endpoint_t;
 
 typedef enum ompt_dispatch_t {
   ompt_dispatch_iteration             = 1,
-  ompt_dispatch_section               = 2
+  ompt_dispatch_section               = 2,
+  ompt_dispatch_ws_loop_chunk         = 3,
+  ompt_dispatch_taskloop_chunk        = 4,
+  ompt_dispatch_distribute_chunk      = 5
 } ompt_dispatch_t;
 
 typedef enum ompt_sync_region_t {
   ompt_sync_region_barrier                DEPRECATED_51 = 1,
   ompt_sync_region_barrier_implicit       DEPRECATED_51 = 2,
   ompt_sync_region_barrier_explicit       = 3,
   ompt_sync_region_barrier_implementation = 4,
   ompt_sync_region_taskwait               = 5,

 typedef enum ompt_work_t {
   ompt_work_loop               = 1,
   ompt_work_sections           = 2,
   ompt_work_single_executor    = 3,
   ompt_work_single_other       = 4,
   ompt_work_workshare          = 5,
   ompt_work_distribute         = 6,
   ompt_work_taskloop           = 7,
-  ompt_work_scope              = 8
+  ompt_work_scope              = 8,
+  ompt_work_loop_static        = 10,
+  ompt_work_loop_dynamic       = 11,
+  ompt_work_loop_guided        = 12,
+  ompt_work_loop_other         = 13
 } ompt_work_t;
 
 typedef enum ompt_mutex_t {
   ompt_mutex_lock                     = 1,
   ompt_mutex_test_lock                = 2,
   ompt_mutex_nest_lock                = 3,
   ompt_mutex_test_nest_lock           = 4,
   ompt_mutex_critical                 = 5,
typedef struct ompt_record_abstract_t {
   ompt_hwid_t hwid;
 } ompt_record_abstract_t;
 
 typedef struct ompt_dependence_t {
   ompt_data_t variable;
   ompt_dependence_type_t dependence_type;
 } ompt_dependence_t;
 
+typedef struct ompt_dispatch_chunk_t {
+  uint64_t start;
+  uint64_t iterations;
+} ompt_dispatch_chunk_t;
+
 typedef int (*ompt_enumerate_states_t) (
   int current_state,
   int *next_state,
   const char **next_state_name
 );
 
 typedef int (*ompt_enumerate_mutex_impls_t) (
   int current_impl,

 typedef struct ompt_record_parallel_end_t {
   ompt_id_t parallel_id;
   ompt_id_t encountering_task_id;
   int flags;
   const void *codeptr_ra;
 } ompt_record_parallel_end_t;
 
 typedef void (*ompt_callback_work_t) (
-  ompt_work_t wstype,
+  ompt_work_t work_type,
   ompt_scope_endpoint_t endpoint,
   ompt_data_t *parallel_data,
   ompt_data_t *task_data,
   uint64_t count,
   const void *codeptr_ra
 );
 
 typedef struct ompt_record_work_t {
-  ompt_work_t wstype;
+  ompt_work_t work_type;
   ompt_scope_endpoint_t endpoint;
   ompt_id_t parallel_id;
   ompt_id_t task_id;
   uint64_t count;
   const void *codeptr_ra;
 } ompt_record_work_t;
 
 typedef void (*ompt_callback_dispatch_t) (

openmp/runtime/src/kmp_dispatch.cpp

if (status == 0) {                                                           \
     if (ompt_enabled.ompt_callback_work) {                                     \
       ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL);              \
       ompt_task_info_t *task_info = __ompt_get_task_info_object(0);            \
       ompt_callbacks.ompt_callback(ompt_callback_work)(                        \
           ompt_work_loop, ompt_scope_end, &(team_info->parallel_data),         \
           &(task_info->task_data), 0, codeptr);                                \
     }                                                                          \
   }
+#define OMPT_LOOP_DISPATCH(lb, ub, st, status)                                 \
+  if (ompt_enabled.ompt_callback_dispatch && status) {                         \
+    ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL);                \
+    ompt_task_info_t *task_info = __ompt_get_task_info_object(0);              \
+    ompt_dispatch_chunk_t chunk;                                               \
+    ompt_data_t instance = ompt_data_none;                                     \
+    OMPT_GET_DISPATCH_CHUNK(chunk, lb, ub, st);                                \
+    instance.ptr = &chunk;                                                     \
+    ompt_callbacks.ompt_callback(ompt_callback_dispatch)(                      \
+        &(team_info->parallel_data), &(task_info->task_data),                  \
+        ompt_dispatch_ws_loop_chunk, instance);                                \
+  }
 // TODO: implement count
 #else
 #define OMPT_LOOP_END // no-op
+#define OMPT_LOOP_DISPATCH // no-op
 #endif
 
 #if KMP_STATS_ENABLED
 #define KMP_STATS_LOOP_END                                                     \
   {                                                                            \
     kmp_int64 u, l, t, i;                                                      \
     l = (kmp_int64)(*p_lb);                                                    \
     u = (kmp_int64)(*p_ub);                                                    \
#ifdef KMP_DEBUG
       KD_TRACE(10, (buff, gtid, *p_lb, *p_ub, *p_st, p_last,
                     (p_last ? *p_last : 0), status));
       __kmp_str_free(&buff);
     }
 #endif
 #if INCLUDE_SSC_MARKS
     SSC_MARK_DISPATCH_NEXT();
 #endif
+    OMPT_LOOP_DISPATCH(*p_lb, *p_ub, pr->u.p.st, status);
     OMPT_LOOP_END;
     KMP_STATS_LOOP_END;
     return status;
   } else {
     kmp_int32 last = 0;
     dispatch_shared_info_template<T> volatile *sh;
 
     KMP_DEBUG_ASSERT(th->th.th_dispatch ==
#ifdef KMP_DEBUG
     KD_TRACE(10, (buff, gtid, *p_lb, *p_ub, p_st ? *p_st : 0, p_last,
                   (p_last ? *p_last : 0), status));
     __kmp_str_free(&buff);
   }
 #endif
 #if INCLUDE_SSC_MARKS
   SSC_MARK_DISPATCH_NEXT();
 #endif
+  OMPT_LOOP_DISPATCH(*p_lb, *p_ub, pr->u.p.st, status);
   OMPT_LOOP_END;
   KMP_STATS_LOOP_END;
   return status;
 }
 
 template <typename T>
 static void __kmp_dist_get_bounds(ident_t *loc, kmp_int32 gtid,
                                   kmp_int32 *plastiter, T *plower, T *pupper,

openmp/runtime/src/kmp_sched.cpp

 
 #if OMPT_SUPPORT && OMPT_OPTIONAL
   ompt_team_info_t *team_info = NULL;
   ompt_task_info_t *task_info = NULL;
   ompt_work_t ompt_work_type = ompt_work_loop;
 
   static kmp_int8 warn = 0;
 
-  if (ompt_enabled.ompt_callback_work) {
+  if (ompt_enabled.ompt_callback_work || ompt_enabled.ompt_callback_dispatch) {
     // Only fully initialize variables needed by OMPT if OMPT is enabled.
     team_info = __ompt_get_teaminfo(0, NULL);
     task_info = __ompt_get_task_info_object(0);
     // Determine workshare type
     if (loc != NULL) {
       if ((loc->flags & KMP_IDENT_WORK_LOOP) != 0) {
         ompt_work_type = ompt_work_loop;
       } else if ((loc->flags & KMP_IDENT_WORK_SECTIONS) != 0) {
#endif
   KE_TRACE(10, ("__kmpc_for_static_init: T#%d return\n", global_tid));
 
 #if OMPT_SUPPORT && OMPT_OPTIONAL
   if (ompt_enabled.ompt_callback_work) {
     ompt_callbacks.ompt_callback(ompt_callback_work)(
         ompt_work_type, ompt_scope_begin, &(team_info->parallel_data),
         &(task_info->task_data), trip_count, codeptr);
   }
+  if (ompt_enabled.ompt_callback_dispatch) {
+    ompt_dispatch_t dispatch_type;
+    ompt_data_t instance = ompt_data_none;
+    ompt_dispatch_chunk_t dispatch_chunk;
+    if (ompt_work_type == ompt_work_sections) {
+      dispatch_type = ompt_dispatch_section;
+      instance.ptr = codeptr;
+    } else {
+      OMPT_GET_DISPATCH_CHUNK(dispatch_chunk, *plower, *pupper, incr);
+      dispatch_type = (ompt_work_type == ompt_work_distribute)
+                          ? ompt_dispatch_distribute_chunk
+                          : ompt_dispatch_ws_loop_chunk;
+      instance.ptr = &dispatch_chunk;
+    }
+    ompt_callbacks.ompt_callback(ompt_callback_dispatch)(
+        &(team_info->parallel_data), &(task_info->task_data), dispatch_type,
+        instance);
+  }
 #endif
 
   KMP_STATS_LOOP_END(OMP_loop_static_iterations);
   return;
 }
 
 template <typename T>
 static void __kmp_dist_for_static_init(ident_t *loc, kmp_int32 gtid,
                                        kmp_int32 schedule, kmp_int32 *plastiter,
                                        T *plower, T *pupper, T *pupperDist,
                                        typename traits_t<T>::signed_t *pstride,
                                        typename traits_t<T>::signed_t incr,
-                                       typename traits_t<T>::signed_t chunk) {
+                                       typename traits_t<T>::signed_t chunk
+#if OMPT_SUPPORT && OMPT_OPTIONAL
+                                       ,
+                                       void *codeptr
+#endif
+) {
   KMP_COUNT_BLOCK(OMP_DISTRIBUTE);
   KMP_PUSH_PARTITIONED_TIMER(OMP_distribute);
   KMP_PUSH_PARTITIONED_TIMER(OMP_distribute_scheduling);
   typedef typename traits_t<T>::unsigned_t UT;
   typedef typename traits_t<T>::signed_t ST;
   kmp_uint32 tid;
   kmp_uint32 nth;
   kmp_uint32 team_id;
buff = __kmp_str_format(
         "stride=%%%s signed?<%s>\n",
         traits_t<T>::spec, traits_t<T>::spec, traits_t<T>::spec,
         traits_t<ST>::spec, traits_t<T>::spec);
     KD_TRACE(100, (buff, *plastiter, *plower, *pupper, *pupperDist, *pstride));
     __kmp_str_free(&buff);
   }
 #endif
   KE_TRACE(10, ("__kmpc_dist_for_static_init: T#%d return\n", gtid));
+#if OMPT_SUPPORT && OMPT_OPTIONAL
+  if (ompt_enabled.ompt_callback_work || ompt_enabled.ompt_callback_dispatch) {
+    ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL);
+    ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
+    if (ompt_enabled.ompt_callback_work) {
+      ompt_callbacks.ompt_callback(ompt_callback_work)(
+          ompt_work_distribute, ompt_scope_begin, &(team_info->parallel_data),
+          &(task_info->task_data), 0, codeptr);
+    }
+    if (ompt_enabled.ompt_callback_dispatch) {
+      ompt_data_t instance = ompt_data_none;
+      ompt_dispatch_chunk_t dispatch_chunk;
+      OMPT_GET_DISPATCH_CHUNK(dispatch_chunk, *plower, *pupperDist, incr);
+      instance.ptr = &dispatch_chunk;
+      ompt_callbacks.ompt_callback(ompt_callback_dispatch)(
+          &(team_info->parallel_data), &(task_info->task_data),
+          ompt_dispatch_distribute_chunk, instance);
+    }
+  }
+#endif // OMPT_SUPPORT && OMPT_OPTIONAL
   KMP_STATS_LOOP_END(OMP_distribute_iterations);
   return;
 }
 
 template <typename T>
 static void __kmp_team_static_init(ident_t *loc, kmp_int32 gtid,
                                    kmp_int32 *p_last, T *p_lb, T *p_ub,
                                    typename traits_t<T>::signed_t *p_st,
#if OMPT_SUPPORT && OMPT_OPTIONAL
                                     OMPT_GET_RETURN_ADDRESS(0)
 #endif
   );
 }
 /*!
 @}
 */
 
+#if OMPT_SUPPORT && OMPT_OPTIONAL
+#define OMPT_CODEPTR_ARG , OMPT_GET_RETURN_ADDRESS(0)
+#else
+#define OMPT_CODEPTR_ARG
+#endif
+
 /*!
 @ingroup WORK_SHARING
 @param    loc       Source code location
 @param    gtid      Global thread id of this thread
 @param    schedule  Scheduling type for the parallel loop
 @param    plastiter Pointer to the "last iteration" flag
 @param    plower    Pointer to the lower bound
 @param    pupper    Pointer to the upper bound of loop chunk

 @{
 */
 void __kmpc_dist_for_static_init_4(ident_t *loc, kmp_int32 gtid,
                                    kmp_int32 schedule, kmp_int32 *plastiter,
                                    kmp_int32 *plower, kmp_int32 *pupper,
                                    kmp_int32 *pupperD, kmp_int32 *pstride,
                                    kmp_int32 incr, kmp_int32 chunk) {
   __kmp_dist_for_static_init<kmp_int32>(loc, gtid, schedule, plastiter, plower,
-                                        pupper, pupperD, pstride, incr, chunk);
+                                        pupper, pupperD, pstride, incr,
+                                        chunk OMPT_CODEPTR_ARG);
 }
 
 /*!
  See @ref __kmpc_dist_for_static_init_4
  */
 void __kmpc_dist_for_static_init_4u(ident_t *loc, kmp_int32 gtid,
                                     kmp_int32 schedule, kmp_int32 *plastiter,
                                     kmp_uint32 *plower, kmp_uint32 *pupper,
                                     kmp_uint32 *pupperD, kmp_int32 *pstride,
                                     kmp_int32 incr, kmp_int32 chunk) {
   __kmp_dist_for_static_init<kmp_uint32>(loc, gtid, schedule, plastiter, plower,
-                                         pupper, pupperD, pstride, incr, chunk);
+                                         pupper, pupperD, pstride, incr,
+                                         chunk OMPT_CODEPTR_ARG);
 }
 
 /*!
  See @ref __kmpc_dist_for_static_init_4
  */
 void __kmpc_dist_for_static_init_8(ident_t *loc, kmp_int32 gtid,
                                    kmp_int32 schedule, kmp_int32 *plastiter,
                                    kmp_int64 *plower, kmp_int64 *pupper,
                                    kmp_int64 *pupperD, kmp_int64 *pstride,
                                    kmp_int64 incr, kmp_int64 chunk) {
   __kmp_dist_for_static_init<kmp_int64>(loc, gtid, schedule, plastiter, plower,
-                                        pupper, pupperD, pstride, incr, chunk);
+                                        pupper, pupperD, pstride, incr,
+                                        chunk OMPT_CODEPTR_ARG);
 }
 
 /*!
  See @ref __kmpc_dist_for_static_init_4
  */
 void __kmpc_dist_for_static_init_8u(ident_t *loc, kmp_int32 gtid,
                                     kmp_int32 schedule, kmp_int32 *plastiter,
                                     kmp_uint64 *plower, kmp_uint64 *pupper,
                                     kmp_uint64 *pupperD, kmp_int64 *pstride,
                                     kmp_int64 incr, kmp_int64 chunk) {
   __kmp_dist_for_static_init<kmp_uint64>(loc, gtid, schedule, plastiter, plower,
-                                         pupper, pupperD, pstride, incr, chunk);
+                                         pupper, pupperD, pstride, incr,
+                                         chunk OMPT_CODEPTR_ARG);
 }
 /*!
 @}
 */
 
 //------------------------------------------------------------------------------
 // Auxiliary routines for Distribute Parallel Loop construct implementation
 //    Transfer call to template< type T >

openmp/runtime/src/kmp_tasking.cpp

#if KMP_STATS_ENABLED
     }
 #endif // KMP_STATS_ENABLED
 
 // OMPT task begin
 #if OMPT_SUPPORT
     if (UNLIKELY(ompt_enabled.enabled))
       __ompt_task_start(task, current_task, gtid);
 #endif
+#if OMPT_SUPPORT && OMPT_OPTIONAL
+    if (UNLIKELY(ompt_enabled.ompt_callback_dispatch &&
+                 taskdata->ompt_task_info.dispatch_chunk.iterations > 0)) {
+      ompt_data_t instance = ompt_data_none;
+      instance.ptr = &(taskdata->ompt_task_info.dispatch_chunk);
+      ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL);
+      ompt_callbacks.ompt_callback(ompt_callback_dispatch)(
+          &(team_info->parallel_data), &(taskdata->ompt_task_info.task_data),
+          ompt_dispatch_taskloop_chunk, instance);
+      taskdata->ompt_task_info.dispatch_chunk = {0, 0};
+    }
+#endif // OMPT_SUPPORT && OMPT_OPTIONAL
 
 #if OMPD_SUPPORT
     if (ompd_state & OMPD_ENABLE_BP)
       ompd_bp_task_begin();
 #endif
 
 #if USE_ITT_BUILD && USE_ITT_NOTIFY
     kmp_uint64 cur_time;
KA_TRACE(40,
              ("__kmp_taskloop_linear: T#%d; task #%llu: task %p: lower %lld, "
               "upper %lld stride %lld, (offsets %p %p)\n",
               gtid, i, next_task, lower, upper, st,
               next_task_bounds.get_lower_offset(),
               next_task_bounds.get_upper_offset()));
 #if OMPT_SUPPORT
     __kmp_omp_taskloop_task(NULL, gtid, next_task,
                             codeptr_ra); // schedule new task
+#if OMPT_OPTIONAL
+    if (ompt_enabled.ompt_callback_dispatch) {
+      OMPT_GET_DISPATCH_CHUNK(next_taskdata->ompt_task_info.dispatch_chunk,
+                              lower, upper, st);
+    }
+#endif // OMPT_OPTIONAL
 #else
     __kmp_omp_task(gtid, next_task, true); // schedule new task
 #endif
     lower = upper + st; // adjust lower bound for the next iteration
   }
   // free the pattern task and exit
   __kmp_task_start(gtid, task, current_task); // make internal bookkeeping
   // do not execute the pattern task, just do internal bookkeeping

openmp/runtime/src/ompt-event-specific.h

 #define ompt_callback_nest_lock_implemented ompt_event_MAY_ALWAYS_OPTIONAL
 
 #define ompt_callback_flush_implemented ompt_event_MAY_ALWAYS_OPTIONAL
 
 #define ompt_callback_cancel_implemented ompt_event_MAY_ALWAYS_OPTIONAL
 
 #define ompt_callback_reduction_implemented ompt_event_MAY_ALWAYS_OPTIONAL
 
-#define ompt_callback_dispatch_implemented ompt_event_UNIMPLEMENTED
+#define ompt_callback_dispatch_implemented ompt_event_MAY_ALWAYS_OPTIONAL
 
 #define ompt_callback_error_implemented ompt_event_MAY_ALWAYS_OPTIONAL
 
 #endif

openmp/runtime/src/ompt-internal.h

((info->td_flags.task_serial || info->td_flags.tasking_ser)                  \
       (info->td_flags.final ? ompt_task_final : 0x0) |                         \
       (info->td_flags.merged_if0 ? ompt_task_mergeable : 0x0)
 
 typedef struct {
   ompt_frame_t frame;
   ompt_data_t task_data;
   struct kmp_taskdata *scheduling_parent;
   int thread_num;
+  ompt_dispatch_chunk_t dispatch_chunk = {0, 0};
 } ompt_task_info_t;
 
 typedef struct {
   ompt_data_t parallel_data;
   void *master_return_address;
 } ompt_team_info_t;
 
 typedef struct ompt_lw_taskteam_s {

openmp/runtime/src/ompt-specific.h

#define OMPT_STORE_RETURN_ADDRESS(gtid)                                        \
   OmptReturnAddressGuard ReturnAddressGuard{gtid, __builtin_return_address(0)};
 #define OMPT_LOAD_RETURN_ADDRESS(gtid) __ompt_load_return_address(gtid)
 #define OMPT_LOAD_OR_GET_RETURN_ADDRESS(gtid)                                  \
   ((ompt_enabled.enabled && gtid >= 0 && __kmp_threads[gtid] &&                \
     __kmp_threads[gtid]->th.ompt_thread_info.return_address)                   \
        ? __ompt_load_return_address(gtid)                                      \
        : __builtin_return_address(0))
 
+#define OMPT_GET_DISPATCH_CHUNK(chunk, lb, ub, incr)                           \
+  do {                                                                         \
+    if (incr > 0) {                                                            \
+      chunk.start = static_cast<uint64_t>(lb);                                 \
+      chunk.iterations = static_cast<uint64_t>(((ub) - (lb)) / (incr) + 1);    \
+    } else {                                                                   \
+      chunk.start = static_cast<uint64_t>(ub);                                 \
+      chunk.iterations = static_cast<uint64_t>(((lb) - (ub)) / -(incr) + 1);   \
+    }                                                                          \
+  } while (0)
+
 //******************************************************************************
 // inline functions
 //******************************************************************************
 
 inline kmp_info_t *ompt_get_thread_gtid(int gtid) {
   return (gtid >= 0) ? __kmp_thread_from_gtid(gtid) : NULL;
 }
 

openmp/runtime/test/ompt/callback.h

on_ompt_callback_work(
   uint64_t count,
   const void *codeptr_ra)
 {
   switch(endpoint)
   {
     case ompt_scope_begin:
       switch(wstype)
       {
         case ompt_work_loop:
+        case ompt_work_loop_static:
+        case ompt_work_loop_dynamic:
+        case ompt_work_loop_guided:
+        case ompt_work_loop_other:
+        // TODO: add schedule attribute for the different work_loop types.
+        // e.g., ", schedule=%s", ..., ompt_schedule_values[wstype]
           printf("%" PRIu64 ":" _TOOL_PREFIX
                  " ompt_event_loop_begin: parallel_id=%" PRIu64
                  ", parent_task_id=%" PRIu64 ", codeptr_ra=%p, count=%" PRIu64
                  "\n",
                  ompt_get_thread_data()->value, parallel_data->value,
                  task_data->value, codeptr_ra, count);
           break;
         case ompt_work_sections:
           printf("%" PRIu64 ":" _TOOL_PREFIX
                  " ompt_event_sections_begin: parallel_id=%" PRIu64
                  ", parent_task_id=%" PRIu64 ", codeptr_ra=%p, count=%" PRIu64
                  "\n",
                  ompt_get_thread_data()->value, parallel_data->value,
                  task_data->value, codeptr_ra, count);
case ompt_scope_begin:
                  task_data->value, codeptr_ra, count);
           break;
       }
       break;
     case ompt_scope_end:
       switch(wstype)
       {
         case ompt_work_loop:
+        case ompt_work_loop_static:
+        case ompt_work_loop_dynamic:
+        case ompt_work_loop_guided:
+        case ompt_work_loop_other:
           printf("%" PRIu64 ":" _TOOL_PREFIX
                  " ompt_event_loop_end: parallel_id=%" PRIu64
                  ", task_id=%" PRIu64 ", codeptr_ra=%p, count=%" PRIu64 "\n",
                  ompt_get_thread_data()->value, parallel_data->value,
                  task_data->value, codeptr_ra, count);
           break;
         case ompt_work_sections:
           printf("%" PRIu64 ":" _TOOL_PREFIX
case ompt_scope_end:
       }
       break;
     case ompt_scope_beginend:
       printf("ompt_scope_beginend should never be passed to %s\n", __func__);
       exit(-1);
   }
 }
 
+static void on_ompt_callback_dispatch(
+    ompt_data_t *parallel_data,
+    ompt_data_t *task_data,
+    ompt_dispatch_t kind,
+    ompt_data_t instance) {
+  char *event_name = NULL;
+  void *codeptr_ra = NULL;
+  ompt_dispatch_chunk_t *dispatch_chunk = NULL;
+  switch (kind) {
+  case ompt_dispatch_section:
+    event_name = "ompt_event_section_begin";
+    codeptr_ra = instance.ptr;
+    break;
+  case ompt_dispatch_ws_loop_chunk:
+    event_name = "ompt_event_ws_loop_chunk_begin";
+    dispatch_chunk = (ompt_dispatch_chunk_t *)instance.ptr;
+    break;
+  case ompt_dispatch_taskloop_chunk:
+    event_name = "ompt_event_taskloop_chunk_begin";
+    dispatch_chunk = (ompt_dispatch_chunk_t *)instance.ptr;
+    break;
+  case ompt_dispatch_distribute_chunk:
+    event_name = "ompt_event_distribute_chunk_begin";
+    dispatch_chunk = (ompt_dispatch_chunk_t *)instance.ptr;
+    break;
+  default:
+    event_name = "ompt_ws_loop_iteration_begin";
+  }
+  printf("%" PRIu64 ":" _TOOL_PREFIX
+         " %s: parallel_id=%" PRIu64 ", task_id=%" PRIu64
+         ", codeptr_ra=%p, chunk_start=%" PRIu64 ", chunk_iterations=%" PRIu64
+         "\n", ompt_get_thread_data()->value, event_name, parallel_data->value,
+         task_data->value, codeptr_ra,
+         dispatch_chunk ? dispatch_chunk->start : 0,
+         dispatch_chunk ? dispatch_chunk->iterations : 0);
+}
+
 static void on_ompt_callback_masked(ompt_scope_endpoint_t endpoint,
                                     ompt_data_t *parallel_data,
                                     ompt_data_t *task_data,
                                     const void *codeptr_ra) {
   switch(endpoint)
   {
     case ompt_scope_begin:
       printf("%" PRIu64 ":" _TOOL_PREFIX
int ompt_initialize(
   register_ompt_callback_t(ompt_callback_reduction, ompt_callback_sync_region_t);
   register_ompt_callback(ompt_callback_control_tool);
   register_ompt_callback(ompt_callback_flush);
   register_ompt_callback(ompt_callback_cancel);
   register_ompt_callback(ompt_callback_implicit_task);
   register_ompt_callback_t(ompt_callback_lock_init, ompt_callback_mutex_acquire_t);
   register_ompt_callback_t(ompt_callback_lock_destroy, ompt_callback_mutex_t);
   register_ompt_callback(ompt_callback_work);
+  register_ompt_callback(ompt_callback_dispatch);
   register_ompt_callback(ompt_callback_masked);
   register_ompt_callback(ompt_callback_parallel_begin);
   register_ompt_callback(ompt_callback_parallel_end);
   register_ompt_callback(ompt_callback_task_create);
   register_ompt_callback(ompt_callback_task_schedule);
   register_ompt_callback(ompt_callback_dependences);
   register_ompt_callback(ompt_callback_task_dependence);
   register_ompt_callback(ompt_callback_thread_begin);

openmp/runtime/test/ompt/tasks/taskloop_dispatch.c

+// RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
+// REQUIRES: ompt
+// UNSUPPORTED: gnu, intel-16.0
+
+#include "callback.h"
+#include <omp.h>
+
+int main() {
+  unsigned int i, x;
+
+#pragma omp parallel num_threads(2)
+  {
+#pragma omp barrier
+
+#pragma omp master
+#pragma omp taskloop grainsize(4)
+    for (i = 0; i < 16; i++) {
+      // Make every iteration takes at least 1ms
+      delay(1000);
+    }
+  }
+  // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
+
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin:
+  // CHECK-SAME: parent_task_id={{[0-9]+}}
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-9]+]]
+  // CHECK-SAME: requested_team_size=2
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin:
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]]
+  // CHECK-SAME: task_id=[[IMPLICIT_TASK_ID1:[0-9]+]]
+  // CHECK-SAME: team_size=2, thread_num=0
+
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_taskloop_begin:
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]]
+  // CHECK-SAME: parent_task_id=[[IMPLICIT_TASK_ID1]]
+  // CHECK-SAME: codeptr_ra=[[RETURN_ADDRESS:0x[0-f]+]], count=16
+
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create:
+  // CHECK-SAME: new_task_id=[[TASK_ID0:[0-9]+]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create:
+  // CHECK-SAME: new_task_id=[[TASK_ID1:[0-9]+]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create:
+  // CHECK-SAME: new_task_id=[[TASK_ID2:[0-9]+]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create:
+  // CHECK-SAME: new_task_id=[[TASK_ID3:[0-9]+]]
+
+  // CHECK-DAG: {{.*}}: ompt_event_taskloop_chunk_begin:{{.*}}task_id=[[TASK_ID0]]{{.*}}chunk_iterations=4
+  // CHECK-DAG: {{.*}}: ompt_event_taskloop_chunk_begin:{{.*}}task_id=[[TASK_ID1]]{{.*}}chunk_iterations=4
+  // CHECK-DAG: {{.*}}: ompt_event_taskloop_chunk_begin:{{.*}}task_id=[[TASK_ID2]]{{.*}}chunk_iterations=4
+  // CHECK-DAG: {{.*}}: ompt_event_taskloop_chunk_begin:{{.*}}task_id=[[TASK_ID3]]{{.*}}chunk_iterations=4
+
+  return 0;
+}

openmp/runtime/test/ompt/teams/distribute_dispatch.c

+// RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
+// REQUIRES: ompt
+#include "callback.h"
+
+#define WORK_SIZE 64
+
+int main() {
+  int i;
+#pragma omp teams num_teams(4) thread_limit(1)
+#pragma omp distribute dist_schedule(static, WORK_SIZE / 4)
+  for (i = 0; i < WORK_SIZE; i++) {}
+
+  return 0;
+}
+
+// CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_work'
+// CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_dispatch'
+
+// CHECK: 0: NULL_POINTER=[[NULL:.*$]]
+
+// CHECK: {{^}}[[THREAD_ID0:[0-9]+]]: ompt_event_distribute_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID0:[0-9]+]]
+// CHECK-SAME: parent_task_id=[[TASK_ID0:[0-9]+]]
+// CHECK: {{^}}[[THREAD_ID0]]: ompt_event_distribute_chunk_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID0]], task_id=[[TASK_ID0]]
+// CHECK-SAME: chunk_start={{[0-9]+}}, chunk_iterations=16
+
+// CHECK: {{^}}[[THREAD_ID1:[0-9]+]]: ompt_event_distribute_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID1:[0-9]+]]
+// CHECK-SAME: parent_task_id=[[TASK_ID1:[0-9]+]]
+// CHECK: {{^}}[[THREAD_ID1]]: ompt_event_distribute_chunk_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID1]], task_id=[[TASK_ID1]]
+// CHECK-SAME: chunk_start={{[0-9]+}}, chunk_iterations=16
+
+// CHECK: {{^}}[[THREAD_ID2:[0-9]+]]: ompt_event_distribute_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID2:[0-9]+]]
+// CHECK-SAME: parent_task_id=[[TASK_ID2:[0-9]+]]
+// CHECK: {{^}}[[THREAD_ID2]]: ompt_event_distribute_chunk_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID2]], task_id=[[TASK_ID2]]
+// CHECK-SAME: chunk_start={{[0-9]+}}, chunk_iterations=16
+
+// CHECK: {{^}}[[THREAD_ID3:[0-9]+]]: ompt_event_distribute_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID3:[0-9]+]]
+// CHECK-SAME: parent_task_id=[[TASK_ID3:[0-9]+]]
+// CHECK: {{^}}[[THREAD_ID3]]: ompt_event_distribute_chunk_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID3]], task_id=[[TASK_ID3]]
+// CHECK-SAME: chunk_start={{[0-9]+}}, chunk_iterations=16

openmp/runtime/test/ompt/worksharing/for/loop_dispatch.c

+// RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
+// REQUIRES: ompt
+// XFAIL: gcc
+// GCC doesn't call runtime for static schedule
+
+#include "callback.h"
+
+#define WORK_SIZE 64
+
+int main() {
+  int i;
+
+#pragma omp parallel num_threads(4)
+  {
+#pragma omp for schedule(static, WORK_SIZE / 4)
+    for (i = 0; i < WORK_SIZE; i++) {}
+
+#pragma omp for schedule(dynamic)
+    for (i = 0; i < WORK_SIZE; i++) {
+      // Make every iteration takes at least 1ms.
+      delay(1000);
+    }
+
+#pragma omp for schedule(guided)
+    for (i = 0; i < WORK_SIZE; i++) {
+      // Make every iteration takes at least 1ms.
+      delay(1000);
+    }
+  }
+
+  return 0;
+}
+
+// CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_parallel_begin'
+// CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_implicit_task'
+// CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_work'
+// CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_dispatch'
+
+// CHECK: 0: NULL_POINTER=[[NULL:.*$]]
+// CHECK: {{^}}[[THREAD_ID0:[0-9]+]]: ompt_event_parallel_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-9]+]]
+
+// Each thread should have at least one ws-loop-chunk-begin event for each
+// for loop.
+
+// CHECK: {{^}}[[THREAD_ID0]]: ompt_event_implicit_task_begin:
+// CHECK-SAME: task_id=[[TASK_ID0:[0-9]+]]
+// CHECK: {{^}}[[THREAD_ID0]]: ompt_event_loop_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID]], parent_task_id=[[TASK_ID0]]
+// CHECK: {{^}}[[THREAD_ID0]]: ompt_event_ws_loop_chunk_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID0]]
+// CHECK-SAME: chunk_start={{[0-9]+}}, chunk_iterations=16
+// CHECK: {{^}}[[THREAD_ID0]]: ompt_event_loop_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID]], parent_task_id=[[TASK_ID0]]
+// CHECK: {{^}}[[THREAD_ID0]]: ompt_event_ws_loop_chunk_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID0]]
+// CHECK-SAME: chunk_start={{[0-9]+}}, chunk_iterations=1
+// CHECK: {{^}}[[THREAD_ID0]]: ompt_event_loop_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID]], parent_task_id=[[TASK_ID0]]
+// CHECK: {{^}}[[THREAD_ID0]]: ompt_event_ws_loop_chunk_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID0]]
+// CHECK-SAME: chunk_start={{[0-9]+}}, chunk_iterations={{[1-9][0-9]*}}
+
+// CHECK: {{^}}[[THREAD_ID1:[0-9]+]]: ompt_event_implicit_task_begin:
+// CHECK-SAME: task_id=[[TASK_ID1:[0-9]+]]
+// CHECK: {{^}}[[THREAD_ID1]]: ompt_event_loop_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID]], parent_task_id=[[TASK_ID1]]
+// CHECK: {{^}}[[THREAD_ID1]]: ompt_event_ws_loop_chunk_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID1]]
+// CHECK-SAME: chunk_start={{[0-9]+}}, chunk_iterations=16
+// CHECK: {{^}}[[THREAD_ID1]]: ompt_event_loop_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID]], parent_task_id=[[TASK_ID1]]
+// CHECK: {{^}}[[THREAD_ID1]]: ompt_event_ws_loop_chunk_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID1]]
+// CHECK-SAME: chunk_start={{[0-9]+}}, chunk_iterations=1
+// CHECK: {{^}}[[THREAD_ID1]]: ompt_event_loop_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID]], parent_task_id=[[TASK_ID1]]
+// CHECK: {{^}}[[THREAD_ID1]]: ompt_event_ws_loop_chunk_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID1]]
+// CHECK-SAME: chunk_start={{[0-9]+}}, chunk_iterations={{[1-9][0-9]*}}
+
+// CHECK: {{^}}[[THREAD_ID2:[0-9]+]]: ompt_event_implicit_task_begin:
+// CHECK-SAME: task_id=[[TASK_ID2:[0-9]+]]
+// CHECK: {{^}}[[THREAD_ID2]]: ompt_event_loop_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID]], parent_task_id=[[TASK_ID2]]
+// CHECK: {{^}}[[THREAD_ID2]]: ompt_event_ws_loop_chunk_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID2]]
+// CHECK-SAME: chunk_start={{[0-9]+}}, chunk_iterations=16
+// CHECK: {{^}}[[THREAD_ID2]]: ompt_event_loop_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID]], parent_task_id=[[TASK_ID2]]
+// CHECK: {{^}}[[THREAD_ID2]]: ompt_event_ws_loop_chunk_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID2]]
+// CHECK-SAME: chunk_start={{[0-9]+}}, chunk_iterations=1
+// CHECK: {{^}}[[THREAD_ID2]]: ompt_event_loop_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID]], parent_task_id=[[TASK_ID2]]
+// CHECK: {{^}}[[THREAD_ID2]]: ompt_event_ws_loop_chunk_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID2]]
+// CHECK-SAME: chunk_start={{[0-9]+}}, chunk_iterations={{[1-9][0-9]*}}
+
+// CHECK: {{^}}[[THREAD_ID3:[0-9]+]]: ompt_event_implicit_task_begin:
+// CHECK-SAME: task_id=[[TASK_ID3:[0-9]+]]
+// CHECK: {{^}}[[THREAD_ID3]]: ompt_event_loop_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID]], parent_task_id=[[TASK_ID3]]
+// CHECK: {{^}}[[THREAD_ID3]]: ompt_event_ws_loop_chunk_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID3]]
+// CHECK-SAME: chunk_start={{[0-9]+}}, chunk_iterations=16
+// CHECK: {{^}}[[THREAD_ID3]]: ompt_event_loop_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID]], parent_task_id=[[TASK_ID3]]
+// CHECK: {{^}}[[THREAD_ID3]]: ompt_event_ws_loop_chunk_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID3]]
+// CHECK-SAME: chunk_start={{[0-9]+}}, chunk_iterations=1
+// CHECK: {{^}}[[THREAD_ID3]]: ompt_event_loop_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID]], parent_task_id=[[TASK_ID3]]
+// CHECK: {{^}}[[THREAD_ID3]]: ompt_event_ws_loop_chunk_begin:
+// CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID3]]
+// CHECK-SAME: chunk_start={{[0-9]+}}, chunk_iterations={{[1-9][0-9]*}}

openmp/runtime/test/ompt/worksharing/sections_dispatch.c

+// RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
+// REQUIRES: ompt
+// UNSUPPORTED: intel-15.0, intel-16.0, intel-17.0, intel-18.0
+// GCC generates code that does not distinguish between sections and loops
+
+#include "callback.h"
+#include <omp.h>
+
+int main()
+{
+  #pragma omp parallel sections num_threads(2)
+  {
+    #pragma omp section
+    {
+      printf("%lu: section 1\n", ompt_get_thread_data()->value);
+    }
+    #pragma omp section
+    {
+      printf("%lu: section 2\n", ompt_get_thread_data()->value);
+    }
+  }
+
+  // Check if libomp supports the callbacks for this test.
+  // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_work'
+  // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_dispatch'
+
+  // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
+
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_sections_begin:
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-9]+]],
+  // CHECK-SAME: parent_task_id=[[TASK_ID:[0-9]+]],
+  // CHECK-SAME: codeptr_ra=[[SECT_BEGIN:0x[0-f]+]], count=2
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_section_begin:
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID]]
+  // CHECK-SAME: codeptr_ra=[[SECT_BEGIN]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_sections_end:
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id={{[0-9]+}},
+  // CHECK-SAME: codeptr_ra=[[SECT_END:0x[0-f]+]]
+
+  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_sections_begin:
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], parent_task_id=[[TASK_ID:[0-9]+]],
+  // CHECK-SAME: codeptr_ra=[[SECT_BEGIN]], count=2
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_section_begin:
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID]]
+  // CHECK-SAME: codeptr_ra=[[SECT_BEGIN]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_sections_end:
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id={{[0-9]+}},
+  // CHECK-SAME: codeptr_ra=[[SECT_END]]
+
+  return 0;
+}