[OpenMP] Implement printing TDGs to dot files

[Munesanz]

Summary

This patch implements the "__kmp_print_tdg_dot" function, that prints a task dependency graph into a dot file containing the tasks and their dependencies.

It is activated through a new environment variable "KMP_TDG_DOT"

[josemonsalve2]

openmp/runtime/src/kmp_tasking.cpp

5482–5483

Just a suggestion.

[tlwilmar]

I think OMP_ prefix is reserved for environment variables defined by the spec, no? Or is this being added to the spec?

[josemonsalve2]

In D150962#4356578, @tlwilmar wrote:

I think OMP_ prefix is reserved for environment variables defined by the spec, no? Or is this being added to the spec?

This is a good point. This was introduced in a previous patch. We can consider renaming. What do you think @jdoerfert @tianshilei1992? OMPX_ or LIBOMP_?

[Munesanz]

Rename "OMP_TDG_DOT" to "KMP_TDG_DOT" and refactor printing.

[tianshilei1992]

Yeah, KMP is good.

[josemonsalve2]

Ping.

[tianshilei1992]

LG but you might want to get a test case.

[josemonsalve2]

@Munesanz once you add that test case, I will land it.

[Munesanz]

Added test

[tianshilei1992]

openmp/runtime/test/tasking/omp_record_replay_print_dot.cpp

74

it might be better to use FileCheck here.

[Munesanz]

Rebase

openmp/runtime/src/kmp.h

typedef struct kmp_tdg_info {
   kmp_bootstrap_lock_t
       graph_lock; // Protect graph attributes when updated via taskloop_recur
   // Taskloop reduction related
   void *rec_taskred_data; // Data to pass to __kmpc_task_reduction_init or
                           // __kmpc_taskred_init
   kmp_int32 rec_num_taskred;
 } kmp_tdg_info_t;
 
+extern int __kmp_tdg_dot;
 extern kmp_int32 __kmp_max_tdgs;
 extern kmp_tdg_info_t **__kmp_global_tdgs;
 extern kmp_int32 __kmp_curr_tdg_idx;
 extern kmp_int32 __kmp_successors_size;
 extern std::atomic<kmp_int32> __kmp_tdg_task_id;
 extern kmp_int32 __kmp_num_tdg;
 #endif
 

openmp/runtime/src/kmp_global.cpp

 
 // Nesting mode
 int __kmp_nesting_mode = 0;
 int __kmp_nesting_mode_nlevels = 1;
 int *__kmp_nesting_nth_level;
 
 #if OMPX_TASKGRAPH
 // TDG record & replay
+int __kmp_tdg_dot = 0;
 kmp_int32 __kmp_max_tdgs = 100;
 kmp_tdg_info_t **__kmp_global_tdgs = NULL;
 kmp_int32 __kmp_curr_tdg_idx =
     0; // Id of the current TDG being recorded or executed
 kmp_int32 __kmp_num_tdg = 0;
 kmp_int32 __kmp_successors_size = 10; // Initial succesor size list for
                                       // recording
 std::atomic<kmp_int32> __kmp_tdg_task_id = 0;
 #endif
 // end of file //
 

openmp/runtime/src/kmp_settings.cpp

static void __kmp_stg_parse_max_tdgs(char const *name, char const *value,
                                      void *data) {
   __kmp_stg_parse_int(name, value, 0, INT_MAX, &__kmp_max_tdgs);
 } // __kmp_stg_parse_max_tdgs
 
 static void __kmp_std_print_max_tdgs(kmp_str_buf_t *buffer, char const *name,
                                      void *data) {
   __kmp_stg_print_int(buffer, name, __kmp_max_tdgs);
 } // __kmp_std_print_max_tdgs
+
+static void __kmp_stg_parse_tdg_dot(char const *name, char const *value,
+                                   void *data) {
+  __kmp_stg_parse_bool(name, value, &__kmp_tdg_dot);
+} // __kmp_stg_parse_tdg_dot
+
+static void __kmp_stg_print_tdg_dot(kmp_str_buf_t *buffer, char const *name,
+                                   void *data) {
+  __kmp_stg_print_bool(buffer, name, __kmp_tdg_dot);
+} // __kmp_stg_print_tdg_dot
 #endif
 
 static void __kmp_stg_parse_num_hidden_helper_threads(char const *name,
                                                       char const *value,
                                                       void *data) {
   __kmp_stg_parse_int(name, value, 0, 16, &__kmp_hidden_helper_threads_num);
   // If the number of hidden helper threads is zero, we disable hidden helper
   // task
#endif
     {"LIBOMP_USE_HIDDEN_HELPER_TASK", __kmp_stg_parse_use_hidden_helper,
      __kmp_stg_print_use_hidden_helper, NULL, 0, 0},
     {"LIBOMP_NUM_HIDDEN_HELPER_THREADS",
      __kmp_stg_parse_num_hidden_helper_threads,
      __kmp_stg_print_num_hidden_helper_threads, NULL, 0, 0},
 #if OMPX_TASKGRAPH
     {"KMP_MAX_TDGS", __kmp_stg_parse_max_tdgs, __kmp_std_print_max_tdgs, NULL,
      0, 0},
+    {"KMP_TDG_DOT", __kmp_stg_parse_tdg_dot, __kmp_stg_print_tdg_dot, NULL, 0, 0},
 #endif
 
 #if OMPT_SUPPORT
     {"OMP_TOOL", __kmp_stg_parse_omp_tool, __kmp_stg_print_omp_tool, NULL, 0,
      0},
     {"OMP_TOOL_LIBRARIES", __kmp_stg_parse_omp_tool_libraries,
      __kmp_stg_print_omp_tool_libraries, NULL, 0, 0},
     {"OMP_TOOL_VERBOSE_INIT", __kmp_stg_parse_omp_tool_verbose_init,

openmp/runtime/src/kmp_tasking.cpp

__kmp_global_tdgs = (kmp_tdg_info_t **)__kmp_allocate(
         sizeof(kmp_tdg_info_t *) * __kmp_max_tdgs);
 
   if ((__kmp_global_tdgs[tdg_id]) &&
       (__kmp_global_tdgs[tdg_id]->tdg_status != KMP_TDG_NONE))
     res = __kmp_global_tdgs[tdg_id];
   return res;
 }
 
+// __kmp_print_tdg_dot: prints the TDG to a dot file
+// tdg:    ID of the TDG
+void __kmp_print_tdg_dot(kmp_tdg_info_t *tdg) {
+  kmp_int32 tdg_id = tdg->tdg_id;
+  KA_TRACE(10, ("__kmp_print_tdg_dot(enter): T#%d tdg_id=%d \n", gtid, tdg_id));
+
+  char file_name[20];
+  sprintf(file_name, "tdg_%d.dot", tdg_id);
+  kmp_safe_raii_file_t tdg_file(file_name, "w");
+
+  kmp_int32 num_tasks = KMP_ATOMIC_LD_RLX(&tdg->num_tasks);
+  fprintf(tdg_file,
+          "digraph TDG {\n"
+          "   compound=true\n"
+          "   subgraph cluster {\n"
+          "      label=TDG_%d\n",
+          tdg_id);
+  for (kmp_int32 i = 0; i < num_tasks; i++) {
+    fprintf(tdg_file, "      %d[style=bold]\n", i);
+  }
+  fprintf(tdg_file, "   }\n");
+  for (kmp_int32 i = 0; i < num_tasks; i++) {
+    kmp_int32 nsuccessors = tdg->record_map[i].nsuccessors;
+    kmp_int32 *successors = tdg->record_map[i].successors;
+    if (nsuccessors > 0) {
+      for (kmp_int32 j = 0; j < nsuccessors; j++)
+        fprintf(tdg_file, "   %d -> %d \n", i, successors[j]);
+    }
+  }
+  fprintf(tdg_file, "}");
+  KA_TRACE(10, ("__kmp_print_tdg_dot(exit): T#%d tdg_id=%d \n", gtid, tdg_id));
+}
+
 // __kmp_start_record: launch the execution of a previous
 // recorded TDG
 // gtid:   Global Thread ID
 // tdg:    ID of the TDG
 void __kmp_exec_tdg(kmp_int32 gtid, kmp_tdg_info_t *tdg) {
   KMP_DEBUG_ASSERT(tdg->tdg_status == KMP_TDG_READY);
   KA_TRACE(10, ("__kmp_exec_tdg(enter): T#%d tdg_id=%d num_roots=%d\n", gtid,
                 tdg->tdg_id, tdg->num_roots));
void __kmp_end_record(kmp_int32 gtid, kmp_tdg_info_t *tdg) {
   }
 
   // Reset predecessor counter
   for (kmp_int32 i = 0; i < this_num_tasks; i++) {
     KMP_ATOMIC_ST_RLX(&this_record_map[i].npredecessors_counter,
                       this_record_map[i].npredecessors);
   }
   KMP_ATOMIC_ST_RLX(&__kmp_tdg_task_id, 0);
+
+  if (__kmp_tdg_dot)
+    __kmp_print_tdg_dot(tdg);
 }
 
 // __kmpc_end_record_task: wrapper around __kmp_end_record to mark
 // the end of recording phase
 //
 // loc_ref:      Source location information
 // gtid:         Global thread ID
 // input_flags:  Flags attached to the graph

openmp/runtime/test/tasking/omp_record_replay_print_dot.cpp

+// REQUIRES: ompx_taskgraph
+// RUN: %libomp-cxx-compile-and-run
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <cassert>
+
+// Compiler-generated code (emulation)
+typedef struct ident {
+    void* dummy;
+} ident_t;
+
+#ifdef __cplusplus
+extern "C" {
+  int __kmpc_global_thread_num(ident_t *);
+  int __kmpc_start_record_task(ident_t *, int, int, int);
+  void __kmpc_end_record_task(ident_t *, int, int , int);
+}
+#endif
+
+void func(int *num_exec) {
+  #pragma omp atomic
+  (*num_exec)++;
+}
+
+std::string tdg_string= "digraph TDG {\n"
+"   compound=true\n"
+"   subgraph cluster {\n"
+"      label=TDG_0\n"
+"      0[style=bold]\n"
+"      1[style=bold]\n"
+"      2[style=bold]\n"
+"      3[style=bold]\n"
+"   }\n"
+"   0 -> 1 \n"
+"   1 -> 2 \n"
+"   1 -> 3 \n"
+"}";
+
+int main() {
+  int num_exec = 0;
+  int x, y;
+
+  setenv("KMP_TDG_DOT","TRUE",1);
+  remove("tdg_0.dot");
+
+  #pragma omp parallel
+  #pragma omp single
+  {
+    int gtid = __kmpc_global_thread_num(nullptr);
+    int res = __kmpc_start_record_task(nullptr, gtid, /* kmp_tdg_flags */ 0, /* tdg_id */ 0);
+    if (res) {
+      #pragma omp task depend(out : x)
+      func(&num_exec);
+      #pragma omp task depend(in : x) depend(out : y)
+      func(&num_exec);
+      #pragma omp task depend(in : y)
+      func(&num_exec);
+      #pragma omp task depend(in : y)
+      func(&num_exec);
+    }
+
+    __kmpc_end_record_task(nullptr, gtid, /* kmp_tdg_flags */ 0, /* tdg_id */ 0);
+  }
+
+  assert(num_exec == 4);
+
+  std::ifstream tdg_file("tdg_0.dot");
+  assert(tdg_file.is_open());
+
+  std::stringstream tdg_file_stream;
+  tdg_file_stream << tdg_file.rdbuf();
+  int equal = tdg_string.compare(tdg_file_stream.str());
+
+  assert(equal == 0);
+
+  std::cout << "Passed" << std::endl;
+  return 0;
+}
+// CHECK: Passed