[OpenMP] Introduce GOMP taskwait depend in the runtime

[jlpeyton]

Summary

This change introduces the GOMP_taskwait_depend() function. It implements
the OpenMP 5.0 feature of #pragma omp taskwait with depend() clause by
wrapping around the __kmpc_omp_wait_deps() function.

[protze.joachim]

The GOMP wrapper makes sense to me.
Several inline comments for the test.

openmp/runtime/test/tasking/omp50_taskwait_depend.c

15

since this is an OpenMP program, shouldn't we use OpenMP atomic store and load?

26

How can we be sure, that not all threads get occupied by these tasks, while none of the other tasks are scheduled?

51

The test succeeds with clang, when I replace this line with

#pragma omp task if(0) depend(inout: a)
{}

Can you please add this copy of the test?

60

b should be loaded with omp atomic

[AndreyChurbanov]

openmp/runtime/test/tasking/omp50_taskwait_depend.c

18

I think the test is going to hang if executed by a single thread. So omp_get_max_threads() should be checked before parallel, and if it is 1 then omp_set_num_threads() could be used to increase number of threads explicitly.

[jlpeyton]

Test was poorly constructed. Second attempt.

The test is three steps:

1. test a single independent task and make sure the #pragma omp taskwait depend() does not wait on that task. Also ensure that a worker thread grabs the task instead of the generating thread.

2. test a single task with irrelevant dependencies and make sure the #pragma omp taskwait depend() does not wait on that task. Also ensure that a worker thread grabs the task instead of the generating thread.

3. test a single dependent task which takes a while and make sure the #pragma omp taskwait depend() actually waits on it.

The test also forces at least two threads in the parallel region, and then rechecks to ensure that at least two threads were actually created for the parallel region.

The #pragma omp task if(0) depend() version of the test will be part of the GOMP fix if0 patch: D87271

[AndreyChurbanov]

LGTM

[protze.joachim]

openmp/runtime/test/tasking/omp50_taskwait_depend.c

4

For the OMPT tests, I prefer to use XFAIL in the situations, where I expect compiler support at some point.
This way, we won't forget to update the support level when codegen is added for clang/icc .

This is what I have in the OMPT test case:

// clang does not yet support taskwait with depend clause
// clang-12 introduced parsing, but no codegen
// update expected result when codegen in clang was added
// XFAIL: clang

[jlpeyton]

Change to XFAIL for clang and icc and add comment in test file.

[protze.joachim]

LGTM as well.

openmp/runtime/src/kmp_ftn_os.h

#define KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_RUNTIME_NEXT                   \
   GOMP_loop_ull_nonmonotonic_runtime_next
 #define KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_RUNTIME_START                  \
   GOMP_loop_ull_nonmonotonic_runtime_start
 #define KMP_API_NAME_GOMP_PARALLEL_LOOP_NONMONOTONIC_RUNTIME                   \
   GOMP_parallel_loop_nonmonotonic_runtime
 #define KMP_API_NAME_GOMP_PARALLEL_LOOP_MAYBE_NONMONOTONIC_RUNTIME             \
   GOMP_parallel_loop_maybe_nonmonotonic_runtime
 #define KMP_API_NAME_GOMP_TEAMS_REG GOMP_teams_reg
+#define KMP_API_NAME_GOMP_TASKWAIT_DEPEND GOMP_taskwait_depend
 
 #endif /* KMP_FTN_OS_H */

openmp/runtime/src/kmp_gsupport.cpp

void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TEAMS_REG)(void (*fn)(void *),
   KA_TRACE(20, ("GOMP_teams_reg: T#%d num_teams=%u thread_limit=%u flag=%u\n",
                 gtid, num_teams, thread_limit, flags));
   __kmpc_push_num_teams(&loc, gtid, num_teams, thread_limit);
   __kmpc_fork_teams(&loc, 2, (microtask_t)__kmp_GOMP_microtask_wrapper, fn,
                     data);
   KA_TRACE(20, ("GOMP_teams_reg exit: T#%d\n", gtid));
 }
 
+void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKWAIT_DEPEND)(void **depend) {
+  MKLOC(loc, "GOMP_taskwait_depend");
+  int gtid = __kmp_entry_gtid();
+  KA_TRACE(20, ("GOMP_taskwait_depend: T#%d\n", gtid));
+  kmp_gomp_depends_info_t gomp_depends(depend);
+  kmp_int32 ndeps = gomp_depends.get_num_deps();
+  kmp_depend_info_t dep_list[ndeps];
+  for (kmp_int32 i = 0; i < ndeps; i++)
+    dep_list[i] = gomp_depends.get_kmp_depend(i);
+  __kmpc_omp_wait_deps(&loc, gtid, ndeps, dep_list, 0, NULL);
+  KA_TRACE(20, ("GOMP_taskwait_depend exit: T#%d\n", gtid));
+}
+
 /* The following sections of code create aliases for the GOMP_* functions, then
    create versioned symbols using the assembler directive .symver. This is only
    pertinent for ELF .so library. The KMP_VERSION_SYMBOL macro is defined in
    kmp_os.h  */
 
 #ifdef KMP_USE_VERSION_SYMBOLS
 // GOMP_1.0 versioned symbols
 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_ATOMIC_END, 10, "GOMP_1.0");
KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_RUNTIME_NEXT, 50,
                    "GOMP_5.0");
 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_RUNTIME_START, 50,
                    "GOMP_5.0");
 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_NONMONOTONIC_RUNTIME, 50,
                    "GOMP_5.0");
 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_MAYBE_NONMONOTONIC_RUNTIME,
                    50, "GOMP_5.0");
 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TEAMS_REG, 50, "GOMP_5.0");
+KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TASKWAIT_DEPEND, 50, "GOMP_5.0");
 
 #endif // KMP_USE_VERSION_SYMBOLS
 
 #ifdef __cplusplus
 } // extern "C"
 #endif // __cplusplus

openmp/runtime/test/tasking/omp50_taskwait_depend.c

+// RUN: %libomp-compile-and-run
+// UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7, gcc-8
+// clang does not yet support taskwait with depend clause
+// clang-12 introduced parsing, but no codegen
+// TODO: update expected result when codegen in clang is added
+// icc does not yet support taskwait with depend clause
+// TODO: update expected result when support for icc is added
+// XFAIL: clang, icc
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <omp.h>
+#include "omp_my_sleep.h"
+
+int a = 0, b = 0;
+int task_grabbed = 0, task_can_proceed = 0;
+int task2_grabbed = 0, task2_can_proceed = 0;
+
+static void wait_on_flag(int *flag) {
+  int flag_value;
+  int timelimit = 30;
+  int secs = 0;
+  do {
+    #pragma omp atomic read
+    flag_value = *flag;
+    my_sleep(1.0);
+    secs++;
+    if (secs == timelimit) {
+      fprintf(stderr, "error: timeout in wait_on_flag()\n");
+      exit(EXIT_FAILURE);
+    }
+  } while (flag_value == 0);
+}
+
+static void signal_flag(int *flag) {
+  #pragma omp atomic
+  (*flag)++;
+}
+
+int main(int argc, char** argv) {
+
+  // Ensure two threads are running
+  int num_threads = omp_get_max_threads();
+  if (num_threads < 2)
+    omp_set_num_threads(2);
+
+  #pragma omp parallel shared(a)
+  {
+    int a_value;
+    // Let us be extra safe here
+    if (omp_get_num_threads() > 1) {
+      #pragma omp single nowait
+      {
+        // Schedule independent child task that
+        // waits to be flagged after sebsequent taskwait depend()
+        #pragma omp task
+        {
+          signal_flag(&task_grabbed);
+          wait_on_flag(&task_can_proceed);
+        }
+        // Let another worker thread grab the task to execute
+        wait_on_flag(&task_grabbed);
+        // This should be ignored since the task above has
+        // no dependency information
+        #pragma omp taskwait depend(inout: a)
+        // Signal the independent task to proceed
+        signal_flag(&task_can_proceed);
+
+        // Schedule child task with dependencies that taskwait does
+        // not care about
+        #pragma omp task depend(inout: b)
+        {
+          signal_flag(&task2_grabbed);
+          wait_on_flag(&task2_can_proceed);
+          #pragma omp atomic
+          b++;
+        }
+        // Let another worker thread grab the task to execute
+        wait_on_flag(&task2_grabbed);
+        // This should be ignored since the task above has
+        // dependency information on b instead of a
+        #pragma omp taskwait depend(inout: a)
+        // Signal the task to proceed
+        signal_flag(&task2_can_proceed);
+
+        // Generate one child task for taskwait
+        #pragma omp task shared(a) depend(inout: a)
+        {
+          my_sleep(1.0);
+          #pragma omp atomic
+          a++;
+        }
+        #pragma omp taskwait depend(inout: a)
+
+        #pragma omp atomic read
+        a_value = a;
+
+        if (a_value != 1) {
+          fprintf(stderr, "error: dependent task was not executed before "
+                          "taskwait finished\n");
+          exit(EXIT_FAILURE);
+        }
+      } // #pragma omp single
+    } // if (num_threads > 1)
+  } // #pragma omp parallel
+
+  return EXIT_SUCCESS;
+}