Add implementation to two OMPT API routines

[hbae]

Summary

This change adds implementation to ompt_finalize_tool() and ompt_get_task_memory().

[protze.joachim]

The code looks good to me, thanks for fixing the implementation.

Would you consider to also add the test?
https://github.com/OpenMPToolsInterface/LLVM-openmp/commit/9fc7407aeb2568ddcd33b69772c48f47ca595f41

The test actually builds on this change:
https://github.com/OpenMPToolsInterface/LLVM-openmp/commit/ba99a76793babdbfdad34e04776e9eb35a7b14cd

[Hahnfeld]

If possible we should also have a test for ompt_finalize_tool.

[protze.joachim]

I think, a test for ompt_finalize_tool could be something along:

#include <callback.h>
int main() {
#pragma omp parallel num_threads(2)
  {
  }
  printf("Before ompt_finalize_tool\n")
  ompt_finalize_tool();
  printf("After ompt_finalize_tool\n")
  return 0;
}

And then test, that all completion events (implicit task-end, thread-end ) are written between Before and After

[hbae]

Added tests for the routines.

[protze.joachim]

Great, thanks!
I like the approach with USE_PRIVATE_TOOL.

[steakhal]

runtime/test/ompt/tasks/task_memory.c

102

This test seems to be flaky. Could you have a look on that why?
Here is my build-bot failure: http://lab.llvm.org:8011/builders/openmp-clang-x86_64-linux-debian/builds/13978/steps/test-openmp/logs/stdio

[protze.joachim]

runtime/test/ompt/tasks/task_memory.c

99

There is a chance, that the first task gets executed before the second task is created.
This line should also have CHECK-DAG.

runtime/src/ompt-general.cpp

OMPT_API_ROUTINE int ompt_get_task_info(int ancestor_level, int *type,
   if (!ompt_enabled.enabled)
     return 0;
   return __ompt_get_task_info_internal(ancestor_level, type, task_data,
                                        task_frame, parallel_data, thread_num);
 }
 
 OMPT_API_ROUTINE int ompt_get_task_memory(void **addr, size_t *size,
                                           int block) {
-  // stub
-  return 0;
+  return __ompt_get_task_memory_internal(addr, size, block);
 }
 
 /*****************************************************************************
  * num_procs
  ****************************************************************************/
 
 OMPT_API_ROUTINE int ompt_get_num_procs(void) {
   // copied from kmp_ftn_entry.h (but modified: OMPT can only be called when

 /*****************************************************************************
  * misc
  ****************************************************************************/
 
 OMPT_API_ROUTINE uint64_t ompt_get_unique_id(void) {
   return __ompt_get_unique_id_internal();
 }
 
-OMPT_API_ROUTINE void ompt_finalize_tool(void) {
-  // stub
-}
+OMPT_API_ROUTINE void ompt_finalize_tool(void) { __kmp_internal_end_atexit(); }
 
 /*****************************************************************************
  * Target
  ****************************************************************************/
 
 OMPT_API_ROUTINE int ompt_get_target_info(uint64_t *device_num,
                                           ompt_id_t *target_id,
                                           ompt_id_t *host_op_id) {

runtime/src/ompt-specific.cpp

if (thread_num) {
         *thread_num = prev_team->t.t_master_tid;
       //        *thread_num = team->t.t_master_tid;
     }
     return info ? 2 : 0;
   }
   return 0;
 }
 
+int __ompt_get_task_memory_internal(void **addr, size_t *size, int blocknum) {
+  if (blocknum != 0)
+    return 0; // support only a single block
+
+  kmp_info_t *thr = ompt_get_thread();
+  kmp_taskdata_t *taskdata = thr->th.th_current_task;
+  kmp_task_t *task = KMP_TASKDATA_TO_TASK(taskdata);
+
+  if (taskdata->td_flags.tasktype != TASK_EXPLICIT)
+    return 0; // support only explicit task
+
+  void *ret_addr;
+  int64_t ret_size = taskdata->td_size_alloc - sizeof(kmp_taskdata_t);
+
+  // kmp_task_t->data1 is an optional member
+  if (taskdata->td_flags.destructors_thunk)
+    ret_addr = &task->data1 + 1;
+  else
+    ret_addr = &task->part_id + 1;
+
+  ret_size -= (char *)(ret_addr) - (char *)(task);
+  if (ret_size < 0)
+    return 0;
+
+  *addr = ret_addr;
+  *size = ret_size;
+  return 1;
+}
+
 //----------------------------------------------------------
 // team support
 //----------------------------------------------------------
 
 void __ompt_team_assign_id(kmp_team_t *team, ompt_data_t ompt_pid) {
   team->t.ompt_team_info.parallel_data = ompt_pid;
 }
 

runtime/test/ompt/callback.h

static void format_task_type(int type, char *buffer) {
   if (type & ompt_task_merged)
     progress += sprintf(progress, "|ompt_task_merged");
 }
 
 static ompt_set_callback_t ompt_set_callback;
 static ompt_get_callback_t ompt_get_callback;
 static ompt_get_state_t ompt_get_state;
 static ompt_get_task_info_t ompt_get_task_info;
+static ompt_get_task_memory_t ompt_get_task_memory;
 static ompt_get_thread_data_t ompt_get_thread_data;
 static ompt_get_parallel_info_t ompt_get_parallel_info;
 static ompt_get_unique_id_t ompt_get_unique_id;
+static ompt_finalize_tool_t ompt_finalize_tool;
 static ompt_get_num_procs_t ompt_get_num_procs;
 static ompt_get_num_places_t ompt_get_num_places;
 static ompt_get_place_proc_ids_t ompt_get_place_proc_ids;
 static ompt_get_place_num_t ompt_get_place_num;
 static ompt_get_partition_place_nums_t ompt_get_partition_place_nums;
 static ompt_get_proc_id_t ompt_get_proc_id;
 static ompt_enumerate_states_t ompt_enumerate_states;
 static ompt_enumerate_mutex_impls_t ompt_enumerate_mutex_impls;
#define print_fuzzy_address_blocks(addr)                                       \
   printf("%" PRIu64 ": fuzzy_address=0x%" PRIx64 " or 0x%" PRIx64              \
          " or 0x%" PRIx64 " or 0x%" PRIx64 " (%p)\n",                          \
          ompt_get_thread_data()->value,                                        \
          ((uint64_t)addr) / FUZZY_ADDRESS_DISCARD_BYTES - 1,                   \
          ((uint64_t)addr) / FUZZY_ADDRESS_DISCARD_BYTES,                       \
          ((uint64_t)addr) / FUZZY_ADDRESS_DISCARD_BYTES + 1,                   \
          ((uint64_t)addr) / FUZZY_ADDRESS_DISCARD_BYTES + 2, addr)
 
+#define register_callback_t(name, type)                                        \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_callback(name) register_callback_t(name, name##_t)
+
+#ifndef USE_PRIVATE_TOOL
 static void
 on_ompt_callback_mutex_acquire(
   ompt_mutex_t kind,
   unsigned int hint,
   unsigned int impl,
   ompt_wait_id_t wait_id,
   const void *codeptr_ra)
 {
on_ompt_callback_control_tool(
   const void *codeptr_ra)
 {
   ompt_frame_t* omptTaskFrame;
   ompt_get_task_info(0, NULL, (ompt_data_t**) NULL, &omptTaskFrame, NULL, NULL);
   printf("%" PRIu64 ": ompt_event_control_tool: command=%" PRIu64 ", modifier=%" PRIu64 ", arg=%p, codeptr_ra=%p, current_task_frame.exit=%p, current_task_frame.reenter=%p \n", ompt_get_thread_data()->value, command, modifier, arg, codeptr_ra, omptTaskFrame->exit_frame.ptr, omptTaskFrame->enter_frame.ptr);
   return 0; //success
 }
 
-#define register_callback_t(name, type)                       \
-do{                                                           \
-  type f_##name = &on_##name;                                 \
-  if (ompt_set_callback(name, (ompt_callback_t)f_##name) ==   \
-      ompt_set_never)                                         \
-    printf("0: Could not register callback '" #name "'\n");   \
-}while(0)
-
-#define register_callback(name) register_callback_t(name, name##_t)
-
 int ompt_initialize(
   ompt_function_lookup_t lookup,
   int initial_device_num,
   ompt_data_t *tool_data)
 {
   ompt_set_callback = (ompt_set_callback_t) lookup("ompt_set_callback");
   ompt_get_callback = (ompt_get_callback_t) lookup("ompt_get_callback");
   ompt_get_state = (ompt_get_state_t) lookup("ompt_get_state");
   ompt_get_task_info = (ompt_get_task_info_t) lookup("ompt_get_task_info");
+  ompt_get_task_memory = (ompt_get_task_memory_t)lookup("ompt_get_task_memory");
   ompt_get_thread_data = (ompt_get_thread_data_t) lookup("ompt_get_thread_data");
   ompt_get_parallel_info = (ompt_get_parallel_info_t) lookup("ompt_get_parallel_info");
   ompt_get_unique_id = (ompt_get_unique_id_t) lookup("ompt_get_unique_id");
+  ompt_finalize_tool = (ompt_finalize_tool_t)lookup("ompt_finalize_tool");
 
   ompt_get_num_procs = (ompt_get_num_procs_t) lookup("ompt_get_num_procs");
   ompt_get_num_places = (ompt_get_num_places_t) lookup("ompt_get_num_places");
   ompt_get_place_proc_ids = (ompt_get_place_proc_ids_t) lookup("ompt_get_place_proc_ids");
   ompt_get_place_num = (ompt_get_place_num_t) lookup("ompt_get_place_num");
   ompt_get_partition_place_nums = (ompt_get_partition_place_nums_t) lookup("ompt_get_partition_place_nums");
   ompt_get_proc_id = (ompt_get_proc_id_t) lookup("ompt_get_proc_id");
   ompt_enumerate_states = (ompt_enumerate_states_t) lookup("ompt_enumerate_states");
ompt_start_tool_result_t* ompt_start_tool(
   const char *runtime_version)
 {
   static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,&ompt_finalize, 0};
   return &ompt_start_tool_result;
 }
 #ifdef __cplusplus
 }
 #endif
+#endif // ifndef USE_PRIVATE_TOOL

runtime/test/ompt/misc/finalize_tool.c

+// RUN: %libomp-compile-and-run | FileCheck %s
+// REQUIRES: ompt
+#include "callback.h"
+
+int main() {
+#pragma omp parallel num_threads(2)
+  {}
+
+  printf("Before ompt_finalize_tool\n");
+  ompt_finalize_tool();
+  printf("After ompt_finalize_tool\n");
+
+  return 0;
+}
+
+// CHECK: 0: NULL_POINTER=[[NULL:.*$]]
+// CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_thread_begin:
+// CHECK-SAME: thread_type=ompt_thread_initial=1
+
+// CHECK: {{^}}[[THREAD_ID]]: ompt_event_parallel_begin
+// CHECK: {{^}}[[THREAD_ID]]: ompt_event_parallel_end
+
+// CHECK: {{^}}Before ompt_finalize_tool
+
+// CHECK: {{^}}[[THREAD_ID]]: ompt_event_thread_end: thread_id=[[THREAD_ID]]
+// CHECK: 0: ompt_event_runtime_shutdown
+
+// CHECK: {{^}}After ompt_finalize_tool

runtime/test/ompt/tasks/task_memory.c

+// RUN: %libomp-compile-and-run | FileCheck %s
+// REQUIRES: ompt
+// UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7
+#define USE_PRIVATE_TOOL 1
+#include "callback.h"
+#include <omp.h>
+
+int main() {
+  int x;
+#pragma omp parallel num_threads(2)
+  {
+#pragma omp master
+    {
+#pragma omp task
+      { x++; }
+#pragma omp task firstprivate(x)
+      { x++; }
+    }
+  }
+
+  return 0;
+}
+
+static void on_ompt_callback_implicit_task(ompt_scope_endpoint_t endpoint,
+                                           ompt_data_t *parallel_data,
+                                           ompt_data_t *task_data,
+                                           unsigned int team_size,
+                                           unsigned int thread_num, int flag) {
+  void *addr = NULL;
+  size_t size = 0;
+  int result = ompt_get_task_memory(&addr, &size, 0);
+  switch (endpoint) {
+  case ompt_scope_begin:
+    task_data->value = ompt_get_unique_id();
+    printf("ompt_event_implicit_task_begin: task_id=%" PRIu64
+           ", memory_addr=%p, memory_size=%lu, result=%d \n",
+           task_data->value, addr, size, result);
+    break;
+  case ompt_scope_end:
+    printf("ompt_event_implicit_task_end: task_id=%" PRIu64
+           ", memory_addr=%p, memory_size=%lu, result=%d \n",
+           task_data->value, addr, size, result);
+    break;
+  }
+}
+
+static void
+on_ompt_callback_task_create(ompt_data_t *encountering_task_data,
+                             const ompt_frame_t *encountering_task_frame,
+                             ompt_data_t *new_task_data, int flags,
+                             int has_dependences, const void *codeptr_ra) {
+  if (flags & ompt_task_initial)
+    return; // not interested in the initial task
+  new_task_data->value = ompt_get_unique_id();
+  void *addr = NULL;
+  size_t size = 0;
+  printf("ompt_event_task_create: task_id=%" PRIu64 "\n", new_task_data->value);
+}
+
+static void on_ompt_callback_task_schedule(ompt_data_t *first_task_data,
+                                           ompt_task_status_t prior_task_status,
+                                           ompt_data_t *second_task_data) {
+  void *addr = NULL;
+  size_t size = 0;
+  int result = ompt_get_task_memory(&addr, &size, 0);
+  printf("ompt_event_task_schedule: task_id=%" PRIu64
+         ", memory_addr=%p, memory_size=%lu, result=%d\n",
+         first_task_data->value, addr, size, result);
+}
+
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+  ompt_get_unique_id = (ompt_get_unique_id_t)lookup("ompt_get_unique_id");
+  ompt_get_task_memory = (ompt_get_task_memory_t)lookup("ompt_get_task_memory");
+
+  register_callback(ompt_callback_implicit_task);
+  register_callback(ompt_callback_task_create);
+  register_callback(ompt_callback_task_schedule);
+  printf("0: NULL_POINTER=%p\n", (void *)NULL);
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+
+// CHECK: {{^}}0: NULL_POINTER=[[NULL:.*$]]
+
+// CHECK: ompt_event_implicit_task_begin: task_id=[[TASK_ID:[0-9]+]]
+// CHECK-SAME: memory_addr=[[NULL]], memory_size=0, result=0
+
+// CHECK: ompt_event_task_create: task_id=[[TASK_ID_0:[0-9]+]]
+// CHECK: ompt_event_task_create: task_id=[[TASK_ID_1:[0-9]+]]
+
+// Expects non-zero address, size, and result
+// CHECK-DAG: ompt_event_task_schedule: task_id=[[TASK_ID_0]],
+// memory_addr=0x{{[0-f]+}}, memory_size={{[1-9][0-9]*}}, result=1
+// CHECK-DAG: ompt_event_task_schedule: task_id=[[TASK_ID_1]],
+// memory_addr=0x{{[0-f]+}}, memory_size={{[1-9][0-9]*}}, result=1
+
+// CHECK: ompt_event_implicit_task_end: task_id=[[TASK_ID]]
+// CHECK-SAME: memory_addr=[[NULL]], memory_size=0, result=0