Fixed intermittent hang on tests with "target teams if(0)" construct with no parallel inside

[AndreyChurbanov]

Summary

Our testing found that the change r294214 has one issue with offload corner-case tests those have the code fragment:

#pragma omp target teams if(target:0)
{
}

where intermittent hang observed. Investigation showed that when master thread executes spin loop waiting for SAFE reap state of the worker thread this worker may be sleeping on a fork barrier. The solution for the problem is to check the flag the worker thread sleeps on, and resume the worker if needed so that it can mark itself as SAFE to reap.

[Hahnfeld]

LGTM. Coud you add a test case on commit? Maybe with a simple call of __kmpc_fork_call with the right parameters to not rely on compiler support?

openmp/trunk/runtime/src/kmp_runtime.cpp

#endif // KMP_NESTED_HOT_TEAMS
     // Do not reset pointer to parent team to NULL for hot teams.
 
     /* if we are non-hot team, release our threads */
     if( ! use_hot_team ) {
         if (__kmp_tasking_mode != tskm_immediate_exec) {
             // Wait for threads to reach reapable state
             for (f = 1; f < team->t.t_nproc; ++f) {
                 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
-                volatile kmp_uint32 *state = &team->t.t_threads[f]->th.th_reap_state;
+                kmp_info_t *th = team->t.t_threads[f];
+                volatile kmp_uint32 *state = &th->th.th_reap_state;
                 while (*state != KMP_SAFE_TO_REAP) {
 #if KMP_OS_WINDOWS
                     // On Windows a thread can be killed at any time, check this
                     DWORD ecode;
-                    if (__kmp_is_thread_alive(team->t.t_threads[f], &ecode))
-                        KMP_CPU_PAUSE();
-                    else
+                    if (!__kmp_is_thread_alive(th, &ecode)) {
                         *state = KMP_SAFE_TO_REAP; // reset the flag for dead thread
-#else
-                    KMP_CPU_PAUSE();
+                        break;
+                    }
 #endif
+                    // first check if thread is sleeping
+                    kmp_flag_64 fl(&th->th.th_bar[bs_forkjoin_barrier].bb.b_go, th);
+                    if (fl.is_sleeping())
+                        fl.resume(__kmp_gtid_from_thread(th));
+                    KMP_CPU_PAUSE();
                 }
             }
 
             // Delete task teams
             int tt_idx;
             for (tt_idx=0; tt_idx<2; ++tt_idx) {
                 kmp_task_team_t *task_team = team->t.t_task_team[tt_idx];
                 if ( task_team != NULL ) {

openmp/trunk/runtime/test/misc_bugs/teams-no-par.c

+// RUN: %libomp-compile-and-run
+//
+// The test checks the teams construct pseudocode executed on host
+//
+
+#include <stdio.h>
+#include <omp.h>
+
+#ifndef N_TEAMS
+#define N_TEAMS 4
+#endif
+#ifndef N_THR
+#define N_THR 3
+#endif
+
+static int err = 0;
+
+// Internal library staff to emulate compiler's code generation:
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+  int reserved_1;
+  int flags;
+  int reserved_2;
+  int reserved_3;
+  char *psource;
+} ident_t;
+
+static ident_t dummy_loc = {0, 2, 0, 0, ";dummyFile;dummyFunc;0;0;;"};
+
+int __kmpc_global_thread_num(void*);
+void __kmpc_push_num_teams(ident_t const*, int, int, int);
+void __kmpc_fork_teams(ident_t const*, int argc, void *microtask, ...);
+
+#ifdef __cplusplus
+}
+#endif
+
+// Outlined entry point:
+void foo(int *gtid, int *tid, int *nt)
+{ // start "serial" execution by master threads of each team
+  if ( nt ) {
+    printf(" team %d, param %d\n", omp_get_team_num(), *nt);
+  } else {
+    printf("ERROR: teams before parallel: gtid, tid: %d %d, bad pointer: %p\n", *gtid, *tid, nt);
+    err++;
+    return;
+  }
+}
+
+int main()
+{
+  int nt = 4;
+  int th = __kmpc_global_thread_num(NULL); // registers initial thread
+  __kmpc_push_num_teams(&dummy_loc, th, N_TEAMS, N_THR);
+  __kmpc_fork_teams(&dummy_loc, 1, &foo, &nt); // pass 1 shared parameter "nt"
+  if (err)
+    printf("failed with %d errors\n",err);
+  else
+    printf("passed\n");
+  return err;
+}