[openmp] Update OMPT declaration and implementation in the runtime

[lechenyu]

Summary

1. Setup OMPT options and targets in libomptarget's CMakeLists.txt.
2. Modify omp-tools.h.var according to the latest OpenMP specification (omp-tools.h.var).
3. Add additional fields into ompt_task_info_t and ompt_thread_info_t to support target-data, target-task-data, and host-op-id (ompt-internal.h).
4. Set corresponding flags correctly when initializing a target task (kmp_tasking.cpp).
5. Add wrappers for target-emi callbacks (ompt-general.cpp).
6. Add callback interfaces for libomptarget in the runtime. The details of OMPT callbacks are transparent to libomptarget and libomptarget only invokes these interfaces when encountering corresponding events (ompt-general.cpp and exports_so.txt).

[lechenyu]

Fix format issues

[hbae]

openmp/runtime/src/include/omp-tools.h.var

320

Is this an extension?
I could not find it in 5.1, TR10, or in any open ticket.

325

Same here.

[protze.joachim]

openmp/runtime/src/include/omp-tools.h.var

320

The use case for this flag is for omp_target_memcpy and similar functions, where it is possible to copy from one device to another device.
Thanks for the reminder to create an issue for the OpenMP spec. I created #3006 for this

[hbae]

openmp/runtime/src/include/omp-tools.h.var

320

OK.

[protze.joachim]

The commit is put on hold as discussed in 10/6 OpenMP in LLVM call. Someone from AMD (@dhruvachak ?) will work on upstreaming an alternative implementation, which will most probably result in reverting D99803 as well.

openmp/libomptarget/src/CMakeLists.txt

set(LIBOMPTARGET_SRC_FILES
   ${CMAKE_CURRENT_SOURCE_DIR}/api.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/device.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/interface.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/rtl.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/omptarget.cpp
 )
 
 if(LIBOMP_OMPT_SUPPORT_IN_LIBOMPTARGET)
-  list(APPEND LIBOMPTARGET_SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/ompt-target.cpp)
-  list(APPEND LIBOMPTARGET_LLVM_INCLUDE_DIRS ${LIBOMP_INCLUDE_DIR})
+  list(APPEND LIBOMPTARGET_LLVM_INCLUDE_DIRS ${LIBOMP_INCLUDE_DIR} ${LIBOMP_SRC_DIR})
 endif()
 
 set(LIBOMPTARGET_SRC_FILES ${LIBOMPTARGET_SRC_FILES} PARENT_SCOPE)
 include_directories(${LIBOMPTARGET_LLVM_INCLUDE_DIRS})
 
 # Build libomptarget library with libdl dependency.
 add_library(omptarget SHARED ${LIBOMPTARGET_SRC_FILES})
 if (OPENMP_ENABLE_LIBOMPTARGET_PROFILING)

openmp/runtime/src/include/omp-tools.h.var

 
 typedef enum ompt_target_data_op_t {
   ompt_target_data_alloc                      = 1,
   ompt_target_data_transfer_to_device         = 2,
   ompt_target_data_transfer_from_device       = 3,
   ompt_target_data_delete                     = 4,
   ompt_target_data_associate                  = 5,
   ompt_target_data_disassociate               = 6,
+  ompt_target_data_transfer                   = 7,
   ompt_target_data_alloc_async                = 17,
   ompt_target_data_transfer_to_device_async   = 18,
   ompt_target_data_transfer_from_device_async = 19,
-  ompt_target_data_delete_async               = 20
+  ompt_target_data_delete_async               = 20,
+  ompt_target_data_transfer_async             = 23
 } ompt_target_data_op_t;
 
 typedef enum ompt_work_t {
   ompt_work_loop               = 1,
   ompt_work_sections           = 2,
   ompt_work_single_executor    = 3,
   ompt_work_single_other       = 4,
   ompt_work_workshare          = 5,

 } ompt_parallel_flag_t;
 
 typedef enum ompt_target_map_flag_t {
   ompt_target_map_flag_to             = 0x01,
   ompt_target_map_flag_from           = 0x02,
   ompt_target_map_flag_alloc          = 0x04,
   ompt_target_map_flag_release        = 0x08,
   ompt_target_map_flag_delete         = 0x10,
-  ompt_target_map_flag_implicit       = 0x20
+  ompt_target_map_flag_implicit       = 0x20,
+  ompt_target_map_flag_always         = 0x40,
+  ompt_target_map_flag_present        = 0x80,
+  ompt_target_map_flag_close          = 0x100,
+  ompt_target_map_flag_shared         = 0x200
 } ompt_target_map_flag_t;
 
 typedef enum ompt_dependence_type_t {
   ompt_dependence_type_in              = 1,
   ompt_dependence_type_out             = 2,
   ompt_dependence_type_inout           = 3,
   ompt_dependence_type_mutexinoutset   = 4,
   ompt_dependence_type_source          = 5,

openmp/runtime/src/kmp_tasking.cpp

static inline void __ompt_task_init(kmp_taskdata_t *task, int tid) {
   // The calls to __ompt_task_init already have the ompt_enabled condition.
   task->ompt_task_info.task_data.value = 0;
   task->ompt_task_info.frame.exit_frame = ompt_data_none;
   task->ompt_task_info.frame.enter_frame = ompt_data_none;
   task->ompt_task_info.frame.exit_frame_flags =
       ompt_frame_runtime | ompt_frame_framepointer;
   task->ompt_task_info.frame.enter_frame_flags =
       ompt_frame_runtime | ompt_frame_framepointer;
+  task->ompt_task_info.target_data.value = 0;
+  task->ompt_task_info.is_target_task = false;
 }
 
 // __ompt_task_start:
 //   Build and trigger task-begin event
 static inline void __ompt_task_start(kmp_task_t *task,
                                      kmp_taskdata_t *current_task,
                                      kmp_int32 gtid) {
   kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
kmp_task_t *__kmpc_omp_target_task_alloc(ident_t *loc_ref, kmp_int32 gtid,
                                          size_t sizeof_shareds,
                                          kmp_routine_entry_t task_entry,
                                          kmp_int64 device_id) {
   if (__kmp_enable_hidden_helper) {
     auto &input_flags = reinterpret_cast<kmp_tasking_flags_t &>(flags);
     input_flags.hidden_helper = TRUE;
   }
 
-  return __kmpc_omp_task_alloc(loc_ref, gtid, flags, sizeof_kmp_task_t,
-                               sizeof_shareds, task_entry);
+  kmp_task_t *retval = __kmpc_omp_task_alloc(
+      loc_ref, gtid, flags, sizeof_kmp_task_t, sizeof_shareds, task_entry);
+
+#if OMPT_SUPPORT
+  if (UNLIKELY(ompt_enabled.enabled)) {
+    kmp_taskdata_t *task_data = KMP_TASK_TO_TASKDATA(retval);
+    task_data->ompt_task_info.is_target_task = true;
+  }
+#endif
+
+  return retval;
 }
 
 /*!
 @ingroup TASKING
 @param loc_ref location of the original task directive
 @param gtid Global Thread ID of encountering thread
 @param new_task task thunk allocated by __kmpc_omp_task_alloc() for the ''new
 task''
#if OMPT_SUPPORT
   if (UNLIKELY(ompt_enabled.enabled)) {
     if (!new_taskdata->td_flags.started) {
       OMPT_STORE_RETURN_ADDRESS(gtid);
       parent = new_taskdata->td_parent;
       if (!parent->ompt_task_info.frame.enter_frame.ptr) {
         parent->ompt_task_info.frame.enter_frame.ptr =
             OMPT_GET_FRAME_ADDRESS(0);
       }
+      int flag = ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(new_taskdata);
+      if (new_taskdata->ompt_task_info.is_target_task) {
+        flag |= ompt_task_target;
+      }
       if (ompt_enabled.ompt_callback_task_create) {
         ompt_callbacks.ompt_callback(ompt_callback_task_create)(
             &(parent->ompt_task_info.task_data),
             &(parent->ompt_task_info.frame),
-            &(new_taskdata->ompt_task_info.task_data),
-            ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(new_taskdata), 0,
+            &(new_taskdata->ompt_task_info.task_data), flag, 0,
             OMPT_LOAD_RETURN_ADDRESS(gtid));
       }
     } else {
       // We are scheduling the continuation of an UNTIED task.
       // Scheduling back to the parent task.
       __ompt_task_finish(new_task,
                          new_taskdata->ompt_task_info.scheduling_parent,
                          ompt_task_switch);

openmp/runtime/src/ompt-general.cpp

 
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #if KMP_OS_UNIX
 #include <dlfcn.h>
 #endif
+#include "kmp.h"
 
 /*****************************************************************************
  * ompt include files
  ****************************************************************************/
 
 #include "ompt-specific.cpp"
 
 /*****************************************************************************
#undef ompt_event_macro
   return ret;
 }
 
 void ompt_callback_target_data_op_emi_wrapper(
     ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
     ompt_data_t *target_data, ompt_id_t *host_op_id,
     ompt_target_data_op_t optype, void *src_addr, int src_device_num,
     void *dest_addr, int dest_device_num, size_t bytes,
-    const void *codeptr_ra) {}
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin || endpoint == ompt_scope_beginend) {
+    *host_op_id = __ompt_get_unique_id_internal();
+  }
+  if (endpoint == ompt_scope_end || endpoint == ompt_scope_beginend) {
+    ompt_callbacks_noemi.ompt_callback(ompt_callback_target_data_op)(
+        target_data->value, *host_op_id, optype, src_addr, src_device_num,
+        dest_addr, dest_device_num, bytes, codeptr_ra);
+  }
+}
 
 void ompt_callback_target_emi_wrapper(ompt_target_t kind,
                                       ompt_scope_endpoint_t endpoint,
                                       int device_num, ompt_data_t *task_data,
                                       ompt_data_t *target_task_data,
                                       ompt_data_t *target_data,
-                                      const void *codeptr_ra) {}
+                                      const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin || endpoint == ompt_scope_beginend) {
+    target_data->value = __ompt_get_unique_id_internal();
+  }
+  ompt_callbacks_noemi.ompt_callback(ompt_callback_target)(
+      kind, endpoint, device_num, task_data, target_data->value, codeptr_ra);
+}
 
 void ompt_callback_target_map_emi_wrapper(ompt_data_t *target_data,
                                           unsigned int nitems, void **host_addr,
                                           void **device_addr, size_t *bytes,
                                           unsigned int *mapping_flags,
-                                          const void *codeptr_ra) {}
+                                          const void *codeptr_ra) {
+  ompt_callbacks_noemi.ompt_callback(ompt_callback_target_map)(
+      target_data->value, nitems, host_addr, device_addr, bytes, mapping_flags,
+      codeptr_ra);
+}
 
 void ompt_callback_target_submit_emi_wrapper(ompt_scope_endpoint_t endpoint,
                                              ompt_data_t *target_data,
                                              ompt_id_t *host_op_id,
                                              unsigned int requested_num_teams) {
-
+  if (endpoint == ompt_scope_begin || endpoint == ompt_scope_beginend) {
+    ompt_id_t op_id = __ompt_get_unique_id_internal();
+    ompt_callbacks_noemi.ompt_callback(ompt_callback_target_submit)(
+        target_data->value, op_id, requested_num_teams);
+  }
 }
 
 #if KMP_OS_DARWIN
 
 // While Darwin supports weak symbols, the library that wishes to provide a new
 // implementation has to link against this runtime which defeats the purpose
 // of having tools that are agnostic of the underlying runtime implementation.
 //
#define ompt_interface_fn(fn)                                                  \
   fn##_t fn##_f = fn;                                                          \
   if (strcmp(s, #fn) == 0)                                                     \
     return (ompt_interface_fn_t)fn##_f;
 
   FOREACH_OMPT_INQUIRY_FN(ompt_interface_fn)
 
   return NULL;
 }
+
+void __ompt_get_target_data_info(ompt_data_t **task_data,
+                                 ompt_data_t **target_task_data,
+                                 ompt_data_t **target_data, bool *is_nowait) {
+  ompt_data_t *task_data_val, *target_task_data_val, *target_data_val;
+  kmp_info_t *thr = ompt_get_thread();
+  kmp_taskdata *current_task = thr->th.th_current_task;
+  ompt_task_info_t *current_task_info = OMPT_CUR_TASK_INFO(thr);
+  if (current_task_info->is_target_task) {
+    target_task_data_val = &current_task_info->task_data;
+    task_data_val = &current_task->td_parent->ompt_task_info.task_data;
+    target_data_val = &current_task_info->target_data;
+  } else {
+    target_task_data_val = NULL;
+    task_data_val = &current_task_info->task_data;
+    target_data_val = &current_task_info->target_data;
+  }
+  if (task_data) {
+    *task_data = task_data_val;
+  }
+  if (target_task_data) {
+    *target_task_data = target_task_data_val;
+  }
+  if (target_data) {
+    *target_data = target_data_val;
+  }
+  if (is_nowait) {
+    *is_nowait = current_task_info->is_target_task;
+  }
+}
+/*****************************************************************************
+ * Delegation of target-related OMPT callbacks
+ ****************************************************************************/
+_OMP_EXTERN void libomp_ompt_callback_target_emi(ompt_target_t kind,
+                                                 ompt_scope_endpoint_t endpoint,
+                                                 int device_num,
+                                                 void *codeptr) {
+  ompt_data_t *task_data, *target_task_data, *target_data;
+  bool is_nowait;
+  __ompt_get_target_data_info(&task_data, &target_task_data, &target_data,
+                              &is_nowait);
+  if (is_nowait) {
+    codeptr = nullptr;
+  }
+  switch (endpoint) {
+  case ompt_scope_begin:
+    *target_data = ompt_data_none;
+    ompt_target_callbacks.ompt_callback(ompt_callback_target_emi)(
+        kind, endpoint, device_num, task_data, target_task_data, target_data,
+        codeptr);
+    break;
+  case ompt_scope_end:
+    ompt_target_callbacks.ompt_callback(ompt_callback_target_emi)(
+        kind, endpoint, device_num, task_data, target_task_data, target_data,
+        codeptr);
+    *target_data = ompt_data_none;
+    break;
+  }
+}
+
+_OMP_EXTERN void libomp_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, bool ompRoutine, void *codeptr) {
+  ompt_data_t *target_task_data, *target_data;
+  bool is_nowait;
+  __ompt_get_target_data_info(nullptr, &target_task_data, &target_data,
+                              &is_nowait);
+  if (is_nowait) {
+    codeptr = nullptr;
+  }
+  ompt_id_t *host_op_id;
+  kmp_info_t *thr = ompt_get_thread();
+  host_op_id = &thr->th.ompt_thread_info.host_op_id;
+  if (ompRoutine &&
+      (endpoint == ompt_scope_begin || endpoint == ompt_scope_beginend)) {
+    *target_data = ompt_data_none;
+  }
+  ompt_target_callbacks.ompt_callback(ompt_callback_target_data_op_emi)(
+      endpoint, target_task_data, target_data, host_op_id, optype, src_addr,
+      src_device_num, dest_addr, dest_device_num, bytes, codeptr);
+  if (ompRoutine &&
+      (endpoint == ompt_scope_end || endpoint == ompt_scope_beginend)) {
+    *target_data = ompt_data_none;
+  }
+}
+
+_OMP_EXTERN void libomp_ompt_callback_target_map_emi(
+    unsigned int nitems, void **host_addr, void **device_addr, size_t *bytes,
+    unsigned int *mapping_flags, void *codeptr) {
+  ompt_data_t *target_data;
+  bool is_nowait;
+  __ompt_get_target_data_info(nullptr, nullptr, &target_data, &is_nowait);
+  if (is_nowait) {
+    codeptr = nullptr;
+  }
+  ompt_target_callbacks.ompt_callback(ompt_callback_target_map_emi)(
+      target_data, nitems, host_addr, device_addr, bytes, mapping_flags,
+      codeptr);
+}
+
+_OMP_EXTERN void
+libomp_ompt_callback_target_submit_emi(ompt_scope_endpoint_t endpoint,
+                                       unsigned int requested_num_teams) {
+  ompt_data_t *target_data;
+  __ompt_get_target_data_info(nullptr, nullptr, &target_data, nullptr);
+  ompt_id_t *host_op_id;
+  kmp_info_t *thr = ompt_get_thread();
+  host_op_id = &thr->th.ompt_thread_info.host_op_id;
+  ompt_target_callbacks.ompt_callback(ompt_callback_target_submit_emi)(
+      endpoint, target_data, host_op_id, requested_num_teams);
+}

openmp/runtime/src/ompt-internal.h

 /*
  * ompt-internal.h - header of OMPT internal data structures
  */
 
 //===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef __OMPT_INTERNAL_H__
 #define __OMPT_INTERNAL_H__
 
 #include "ompt-event-specific.h"
+#include "ompt-target-api.h"
 #include "omp-tools.h"
 
 #define OMPT_VERSION 1
 
 #define _OMP_EXTERN extern "C"
 
 #define OMPT_INVOKER(x)                                                        \
   ((x == fork_context_gnu) ? ompt_parallel_invoker_program                     \
typedef struct ompt_callbacks_active_s {
   unsigned int enabled : 1;
 #define ompt_event_macro(event, callback, eventid) unsigned int event : 1;
 
   FOREACH_OMPT_HOST_EVENT(ompt_event_macro)
 
 #undef ompt_event_macro
 } ompt_callbacks_active_t;
 
-/* Bitmap to mark OpenMP 5.1 target events as registered*/
-typedef struct ompt_target_callbacks_active_s {
-  unsigned int enabled : 1;
-#define ompt_event_macro(event, callback, eventid) unsigned int event : 1;
-
-  FOREACH_OMPT_51_TARGET_EVENT(ompt_event_macro)
-
-#undef ompt_event_macro
-} ompt_target_callbacks_active_t;
-
 #define TASK_TYPE_DETAILS_FORMAT(info)                                         \
   ((info->td_flags.task_serial || info->td_flags.tasking_ser)                  \
        ? ompt_task_undeferred                                                  \
        : 0x0) |                                                                \
       ((!(info->td_flags.tiedness)) ? ompt_task_untied : 0x0) |                \
       (info->td_flags.final ? ompt_task_final : 0x0) |                         \
       (info->td_flags.merged_if0 ? ompt_task_mergeable : 0x0)
 
 typedef struct {
   ompt_frame_t frame;
   ompt_data_t task_data;
   struct kmp_taskdata *scheduling_parent;
+  ompt_data_t target_data; // ompt_data for the enclosed target region
   int thread_num;
+  bool is_target_task;
 } ompt_task_info_t;
 
 typedef struct {
   ompt_data_t parallel_data;
   void *master_return_address;
 } ompt_team_info_t;
 
 typedef struct ompt_lw_taskteam_s {
   ompt_team_info_t ompt_team_info;
   ompt_task_info_t ompt_task_info;
   int heap;
   struct ompt_lw_taskteam_s *parent;
 } ompt_lw_taskteam_t;
 
 typedef struct {
   ompt_data_t thread_data;
   ompt_data_t task_data; /* stored here from implicit barrier-begin until
                             implicit-task-end */
   void *return_address; /* stored here on entry of runtime */
   ompt_state_t state;
   ompt_wait_id_t wait_id;
   int ompt_task_yielded;
   int parallel_flags; // information for the last parallel region invoked
   void *idle_frame;
+  ompt_id_t host_op_id;
 } ompt_thread_info_t;
 
 extern ompt_callbacks_internal_t ompt_callbacks;
 extern ompt_target_callbacks_internal_t ompt_target_callbacks;
 extern ompt_callbacks_internal_noemi_t ompt_callbacks_noemi;
 
 #if OMPT_SUPPORT && OMPT_OPTIONAL
 #if USE_FAST_MEMORY

openmp/runtime/src/ompt-target-api.h

+
+
+#ifndef OMPT_TARGET_API_H
+#define OMPT_TARGET_API_H
+
+#include "kmp_os.h"
+#include "omp-tools.h"
+
+#define _OMP_EXTERN extern "C"
+
+#ifdef FROM_LIBOMPTARGET
+#define OMPT_INTERFACE_ATTRIBUTE KMP_WEAK_ATTRIBUTE_INTERNAL
+#else
+#define OMPT_INTERFACE_ATTRIBUTE
+#endif
+
+/* Bitmap to mark OpenMP 5.1 target events as registered*/
+typedef struct ompt_target_callbacks_active_s {
+  unsigned int enabled : 1;
+#define ompt_event_macro(event, callback, eventid) unsigned int event : 1;
+
+  FOREACH_OMPT_51_TARGET_EVENT(ompt_event_macro)
+
+#undef ompt_event_macro
+} ompt_target_callbacks_active_t;
+
+_OMP_EXTERN OMPT_INTERFACE_ATTRIBUTE bool
+libomp_start_tool(ompt_target_callbacks_active_t *libomptarget_ompt_enabled);
+
+_OMP_EXTERN OMPT_INTERFACE_ATTRIBUTE void
+libomp_ompt_callback_target_emi(ompt_target_t kind,
+                                ompt_scope_endpoint_t endpoint, int device_num,
+                                void *codeptr);
+
+_OMP_EXTERN OMPT_INTERFACE_ATTRIBUTE void
+libomp_ompt_callback_target_data_op_emi(ompt_scope_endpoint_t endpoint,
+                                        ompt_target_data_op_t optype,
+                                        void *src_addr, int src_device_num,
+                                        void *dest_addr, int dest_device_num,
+                                        size_t bytes, bool ompRoutine,
+                                        void *codeptr);
+
+_OMP_EXTERN OMPT_INTERFACE_ATTRIBUTE void
+libomp_ompt_callback_target_map_emi(unsigned int nitems, void **host_addr,
+                                    void **device_addr, size_t *bytes,
+                                    unsigned int *mapping_flags, void *codeptr);
+
+_OMP_EXTERN OMPT_INTERFACE_ATTRIBUTE void
+libomp_ompt_callback_target_submit_emi(ompt_scope_endpoint_t endpoint,
+                                       unsigned int requested_num_teams);
+
+#endif // OMPT_TARGET_API_H