[OpenMP] Adding GOMP compatible cancellation

[jlpeyton]

Summary

Remove fatal error messages from the cancellation API for GOMP
Add __kmp_barrier_gomp_cancel() to implement cancellation of parallel regions.
This new function uses the linear barrier algorithm with a cancellable nonsleepable wait loop.

[tlwilmar]

LGTM

runtime/src/kmp.h

 KMP_EXPORT void kmpc_free(void *ptr);
 
 /* declarations for internal use */
 
 extern int __kmp_barrier(enum barrier_type bt, int gtid, int is_split,
                          size_t reduce_size, void *reduce_data,
                          void (*reduce)(void *, void *));
 extern void __kmp_end_split_barrier(enum barrier_type bt, int gtid);
+extern int __kmp_barrier_gomp_cancel(int gtid);
 
 /*!
  * Tell the fork call which compiler generated the fork call, and therefore how
  * to deal with the call.
  */
 enum fork_context_e {
   fork_context_gnu, /**< Called from GNU generated code, so must not invoke the
                        microtask internally. */

runtime/src/kmp_barrier.cpp

 #define ngo_sync() ((void)0)
 #endif /* KMP_MIC && USE_NGO_STORES */
 
 void __kmp_print_structure(void); // Forward declaration
 
 // ---------------------------- Barrier Algorithms ----------------------------
 
 // Linear Barrier
-static void __kmp_linear_barrier_gather(
+template <bool cancellable = false>
+static bool __kmp_linear_barrier_gather_template(
     enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
     void (*reduce)(void *, void *) USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
   KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_linear_gather);
   kmp_team_t *team = this_thr->th.th_team;
   kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
   kmp_info_t **other_threads = team->t.t_threads;
 
   KA_TRACE(
#endif /* KMP_CACHE_MANAGE */
                     "arrived(%p) == %llu\n",
                     gtid, team->t.t_id, tid, __kmp_gtid_from_tid(i, team),
                     team->t.t_id, i,
                     &other_threads[i]->th.th_bar[bt].bb.b_arrived, new_state));
 
       // Wait for worker thread to arrive
       kmp_flag_64 flag(&other_threads[i]->th.th_bar[bt].bb.b_arrived,
                        new_state);
+      if (cancellable) {
+        bool cancelled = flag.wait_cancellable_nosleep(
+            this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
+        if (cancelled)
+          return true;
+      } else {
         flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
+      }
       ANNOTATE_BARRIER_END(other_threads[i]);
 #if USE_ITT_BUILD && USE_ITT_NOTIFY
       // Barrier imbalance - write min of the thread time and the other thread
       // time to the thread.
       if (__kmp_forkjoin_frames_mode == 2) {
         this_thr->th.th_bar_min_time = KMP_MIN(
             this_thr->th.th_bar_min_time, other_threads[i]->th.th_bar_min_time);
       }
KA_TRACE(20, ("__kmp_linear_barrier_gather: T#%d(%d:%d) set team %d "
                   "arrived(%p) = %llu\n",
                   gtid, team->t.t_id, tid, team->t.t_id, &team_bar->b_arrived,
                   new_state));
   }
   KA_TRACE(
       20,
       ("__kmp_linear_barrier_gather: T#%d(%d:%d) exit for barrier type %d\n",
        gtid, team->t.t_id, tid, bt));
+  return false;
 }
 
-static void __kmp_linear_barrier_release(
+template <bool cancellable = false>
+static bool __kmp_linear_barrier_release_template(
     enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
     int propagate_icvs USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
   KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_linear_release);
   kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
   kmp_team_t *team;
 
   if (KMP_MASTER_TID(tid)) {
     unsigned int i;
#endif /* KMP_CACHE_MANAGE */
                          other_threads[i]);
         flag.release();
       }
     }
   } else { // Wait for the MASTER thread to release us
     KA_TRACE(20, ("__kmp_linear_barrier_release: T#%d wait go(%p) == %u\n",
                   gtid, &thr_bar->b_go, KMP_BARRIER_STATE_BUMP));
     kmp_flag_64 flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
+    if (cancellable) {
+      bool cancelled = flag.wait_cancellable_nosleep(
+          this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj));
+      if (cancelled) {
+        return true;
+      }
+    } else {
       flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj));
+    }
     ANNOTATE_BARRIER_END(this_thr);
 #if USE_ITT_BUILD && USE_ITT_NOTIFY
     if ((__itt_sync_create_ptr && itt_sync_obj == NULL) || KMP_ITT_DEBUG) {
       // In a fork barrier; cannot get the object reliably (or ITTNOTIFY is
       // disabled)
       itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier, 0, -1);
       // Cancel wait on previous parallel region...
       __kmp_itt_task_starting(itt_sync_obj);
 
       if (bt == bs_forkjoin_barrier && TCR_4(__kmp_global.g.g_done))
-        return;
+        return false;
 
       itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier);
       if (itt_sync_obj != NULL)
         // Call prepare as early as possible for "new" barrier
         __kmp_itt_task_finished(itt_sync_obj);
     } else
 #endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */
         // Early exit for reaping threads releasing forkjoin barrier
         if (bt == bs_forkjoin_barrier && TCR_4(__kmp_global.g.g_done))
-      return;
+      return false;
 // The worker thread may now assume that the team is valid.
 #ifdef KMP_DEBUG
     tid = __kmp_tid_from_gtid(gtid);
     team = __kmp_threads[gtid]->th.th_team;
 #endif
     KMP_DEBUG_ASSERT(team != NULL);
     TCW_4(thr_bar->b_go, KMP_INIT_BARRIER_STATE);
     KA_TRACE(20,
              ("__kmp_linear_barrier_release: T#%d(%d:%d) set go(%p) = %u\n",
               gtid, team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE));
     KMP_MB(); // Flush all pending memory write invalidates.
   }
   KA_TRACE(
       20,
       ("__kmp_linear_barrier_release: T#%d(%d:%d) exit for barrier type %d\n",
        gtid, team->t.t_id, tid, bt));
+  return false;
+}
+
+static void __kmp_linear_barrier_gather(
+    enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
+    void (*reduce)(void *, void *) USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
+  __kmp_linear_barrier_gather_template<false>(
+      bt, this_thr, gtid, tid, reduce USE_ITT_BUILD_ARG(itt_sync_obj));
+}
+
+static bool __kmp_linear_barrier_gather_cancellable(
+    enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
+    void (*reduce)(void *, void *) USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
+  return __kmp_linear_barrier_gather_template<true>(
+      bt, this_thr, gtid, tid, reduce USE_ITT_BUILD_ARG(itt_sync_obj));
+}
+
+static void __kmp_linear_barrier_release(
+    enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
+    int propagate_icvs USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
+  __kmp_linear_barrier_release_template<false>(
+      bt, this_thr, gtid, tid, propagate_icvs USE_ITT_BUILD_ARG(itt_sync_obj));
+}
+
+static bool __kmp_linear_barrier_release_cancellable(
+    enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
+    int propagate_icvs USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
+  return __kmp_linear_barrier_release_template<true>(
+      bt, this_thr, gtid, tid, propagate_icvs USE_ITT_BUILD_ARG(itt_sync_obj));
 }
 
 // Tree barrier
 static void
 __kmp_tree_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr, int gtid,
                           int tid, void (*reduce)(void *, void *)
                                        USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
   KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_tree_gather);
#endif // KMP_BARRIER_ICV_PUSH
   }
   KA_TRACE(20, ("__kmp_hierarchical_barrier_release: T#%d(%d:%d) exit for "
                 "barrier type %d\n",
                 gtid, team->t.t_id, tid, bt));
 }
 
 // End of Barrier Algorithms
 
+// type traits for cancellable value
+// if cancellable is true, then is_cancellable is a normal boolean variable
+// if cancellable is false, then is_cancellable is a compile time constant
+template <bool cancellable> struct is_cancellable {};
+template <> struct is_cancellable<true> {
+  bool value;
+  is_cancellable() : value(false) {}
+  is_cancellable(bool b) : value(b) {}
+  is_cancellable &operator=(bool b) {
+    value = b;
+    return *this;
+  }
+  operator bool() const { return value; }
+};
+template <> struct is_cancellable<false> {
+  is_cancellable &operator=(bool b) { return *this; }
+  constexpr operator bool() const { return false; }
+};
+
 // Internal function to do a barrier.
 /* If is_split is true, do a split barrier, otherwise, do a plain barrier
    If reduce is non-NULL, do a split reduction barrier, otherwise, do a split
    barrier
-   Returns 0 if master thread, 1 if worker thread.  */
-int __kmp_barrier(enum barrier_type bt, int gtid, int is_split,
+   When cancellable = false,
+     Returns 0 if master thread, 1 if worker thread.
+   When cancellable = true
+     Returns 0 if not cancelled, 1 if cancelled.  */
+template <bool cancellable = false>
+static int __kmp_barrier_template(enum barrier_type bt, int gtid, int is_split,
                                   size_t reduce_size, void *reduce_data,
                                   void (*reduce)(void *, void *)) {
   KMP_TIME_PARTITIONED_BLOCK(OMP_plain_barrier);
   KMP_SET_THREAD_STATE_BLOCK(PLAIN_BARRIER);
   int tid = __kmp_tid_from_gtid(gtid);
   kmp_info_t *this_thr = __kmp_threads[gtid];
   kmp_team_t *team = this_thr->th.th_team;
   int status = 0;
+  is_cancellable<cancellable> cancelled;
 #if OMPT_SUPPORT && OMPT_OPTIONAL
   ompt_data_t *my_task_data;
   ompt_data_t *my_parallel_data;
   void *return_address;
 #endif
 
   KA_TRACE(15, ("__kmp_barrier: T#%d(%d:%d) has arrived\n", gtid,
                 __kmp_team_from_gtid(gtid)->t.t_id, __kmp_tid_from_gtid(gtid)));
#if USE_DEBUGGER
     } // if
 #endif /* USE_DEBUGGER */
     if (reduce != NULL) {
       // KMP_DEBUG_ASSERT( is_split == TRUE );  // #C69956
       this_thr->th.th_local.reduce_data = reduce_data;
     }
 
     if (KMP_MASTER_TID(tid) && __kmp_tasking_mode != tskm_immediate_exec)
-      __kmp_task_team_setup(
-          this_thr, team,
-          0); // use 0 to only setup the current team if nthreads > 1
+      // use 0 to only setup the current team if nthreads > 1
+      __kmp_task_team_setup(this_thr, team, 0);
 
+    if (cancellable) {
+      cancelled = __kmp_linear_barrier_gather_cancellable(
+          bt, this_thr, gtid, tid, reduce USE_ITT_BUILD_ARG(itt_sync_obj));
+    } else {
       switch (__kmp_barrier_gather_pattern[bt]) {
       case bp_hyper_bar: {
-      KMP_ASSERT(__kmp_barrier_gather_branch_bits[bt]); // don't set branch bits
-      // to 0; use linear
+        // don't set branch bits to 0; use linear
+        KMP_ASSERT(__kmp_barrier_gather_branch_bits[bt]);
         __kmp_hyper_barrier_gather(bt, this_thr, gtid, tid,
                                    reduce USE_ITT_BUILD_ARG(itt_sync_obj));
         break;
       }
       case bp_hierarchical_bar: {
-      __kmp_hierarchical_barrier_gather(bt, this_thr, gtid, tid,
-                                        reduce USE_ITT_BUILD_ARG(itt_sync_obj));
+        __kmp_hierarchical_barrier_gather(
+            bt, this_thr, gtid, tid, reduce USE_ITT_BUILD_ARG(itt_sync_obj));
         break;
       }
       case bp_tree_bar: {
-      KMP_ASSERT(__kmp_barrier_gather_branch_bits[bt]); // don't set branch bits
-      // to 0; use linear
+        // don't set branch bits to 0; use linear
+        KMP_ASSERT(__kmp_barrier_gather_branch_bits[bt]);
         __kmp_tree_barrier_gather(bt, this_thr, gtid, tid,
                                   reduce USE_ITT_BUILD_ARG(itt_sync_obj));
         break;
       }
       default: {
         __kmp_linear_barrier_gather(bt, this_thr, gtid, tid,
                                     reduce USE_ITT_BUILD_ARG(itt_sync_obj));
       }
       }
+    }
 
     KMP_MB();
 
     if (KMP_MASTER_TID(tid)) {
       status = 0;
-      if (__kmp_tasking_mode != tskm_immediate_exec) {
+      if (__kmp_tasking_mode != tskm_immediate_exec && !cancelled) {
         __kmp_task_team_wait(this_thr, team USE_ITT_BUILD_ARG(itt_sync_obj));
       }
 #if USE_DEBUGGER
       // Let the debugger know: All threads are arrived and starting leaving the
       // barrier.
       team->t.t_bar[bt].b_team_arrived += 1;
 #endif
 
 #if OMP_40_ENABLED
+      if (__kmp_omp_cancellation) {
         kmp_int32 cancel_request = KMP_ATOMIC_LD_RLX(&team->t.t_cancel_request);
         // Reset cancellation flag for worksharing constructs
-      if (cancel_request == cancel_loop || cancel_request == cancel_sections) {
+        if (cancel_request == cancel_loop ||
+            cancel_request == cancel_sections) {
           KMP_ATOMIC_ST_RLX(&team->t.t_cancel_request, cancel_noreq);
         }
+      }
 #endif
 #if USE_ITT_BUILD
       /* TODO: In case of split reduction barrier, master thread may send
          acquired event early, before the final summation into the shared
          variable is done (final summation can be a long operation for array
          reductions).  */
       if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
         __kmp_itt_barrier_middle(gtid, itt_sync_obj);

 #endif /* USE_ITT_BUILD */
     } else {
       status = 1;
 #if USE_ITT_BUILD
       if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
         __kmp_itt_barrier_middle(gtid, itt_sync_obj);
 #endif /* USE_ITT_BUILD */
     }
-    if (status == 1 || !is_split) {
+    if ((status == 1 || !is_split) && !cancelled) {
+      if (cancellable) {
+        cancelled = __kmp_linear_barrier_release_cancellable(
+            bt, this_thr, gtid, tid, FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
+      } else {
         switch (__kmp_barrier_release_pattern[bt]) {
         case bp_hyper_bar: {
           KMP_ASSERT(__kmp_barrier_release_branch_bits[bt]);
           __kmp_hyper_barrier_release(bt, this_thr, gtid, tid,
                                       FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
           break;
         }
         case bp_hierarchical_bar: {
           __kmp_hierarchical_barrier_release(
               bt, this_thr, gtid, tid, FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
           break;
         }
         case bp_tree_bar: {
           KMP_ASSERT(__kmp_barrier_release_branch_bits[bt]);
           __kmp_tree_barrier_release(bt, this_thr, gtid, tid,
                                      FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
           break;
         }
         default: {
           __kmp_linear_barrier_release(bt, this_thr, gtid, tid,
                                        FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
         }
         }
-      if (__kmp_tasking_mode != tskm_immediate_exec) {
+      }
+      if (__kmp_tasking_mode != tskm_immediate_exec && !cancelled) {
         __kmp_task_team_sync(this_thr, team);
       }
     }
 
 #if USE_ITT_BUILD
     /* GEH: TODO: Move this under if-condition above and also include in
        __kmp_end_split_barrier(). This will more accurately represent the actual
        release time of the threads for split barriers.  */
if (ompt_enabled.ompt_callback_sync_region) {
           my_task_data, return_address);
     }
 #endif
     this_thr->th.ompt_thread_info.state = ompt_state_work_parallel;
   }
 #endif
   ANNOTATE_BARRIER_END(&team->t.t_bar);
 
+  if (cancellable)
+    return (int)cancelled;
   return status;
 }
 
+// Returns 0 if master thread, 1 if worker thread.
+int __kmp_barrier(enum barrier_type bt, int gtid, int is_split,
+                  size_t reduce_size, void *reduce_data,
+                  void (*reduce)(void *, void *)) {
+  return __kmp_barrier_template<>(bt, gtid, is_split, reduce_size, reduce_data,
+                                  reduce);
+}
+
+#if defined(KMP_GOMP_COMPAT)
+// Returns 1 if cancelled, 0 otherwise
+int __kmp_barrier_gomp_cancel(int gtid) {
+  if (__kmp_omp_cancellation) {
+    int cancelled = __kmp_barrier_template<true>(bs_plain_barrier, gtid, FALSE,
+                                                 0, NULL, NULL);
+    if (cancelled) {
+      int tid = __kmp_tid_from_gtid(gtid);
+      kmp_info_t *this_thr = __kmp_threads[gtid];
+      if (KMP_MASTER_TID(tid)) {
+        // Master does not need to revert anything
+      } else {
+        // Workers need to revert their private b_arrived flag
+        this_thr->th.th_bar[bs_plain_barrier].bb.b_arrived -=
+            KMP_BARRIER_STATE_BUMP;
+      }
+    }
+    return cancelled;
+  }
+  __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
+  return FALSE;
+}
+#endif
+
 void __kmp_end_split_barrier(enum barrier_type bt, int gtid) {
   KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_end_split_barrier);
   KMP_SET_THREAD_STATE_BLOCK(PLAIN_BARRIER);
   int tid = __kmp_tid_from_gtid(gtid);
   kmp_info_t *this_thr = __kmp_threads[gtid];
   kmp_team_t *team = this_thr->th.th_team;
 
   ANNOTATE_BARRIER_BEGIN(&team->t.t_bar);

runtime/src/kmp_ftn_os.h

 #define KMP_API_NAME_GOMP_LOOP_ULL_RUNTIME_START GOMP_loop_ull_runtime_start
 #define KMP_API_NAME_GOMP_LOOP_ULL_STATIC_NEXT GOMP_loop_ull_static_next
 #define KMP_API_NAME_GOMP_LOOP_ULL_STATIC_START GOMP_loop_ull_static_start
 
 // All GOMP_3.0 symbols
 #define KMP_API_NAME_GOMP_TASKYIELD GOMP_taskyield
 
 // All GOMP_4.0 symbols
-// TODO: As of 2013-10-14, none of the GOMP_4.0 functions are implemented in
-// libomp
 #define KMP_API_NAME_GOMP_BARRIER_CANCEL GOMP_barrier_cancel
 #define KMP_API_NAME_GOMP_CANCEL GOMP_cancel
 #define KMP_API_NAME_GOMP_CANCELLATION_POINT GOMP_cancellation_point
 #define KMP_API_NAME_GOMP_LOOP_END_CANCEL GOMP_loop_end_cancel
 #define KMP_API_NAME_GOMP_PARALLEL_LOOP_DYNAMIC GOMP_parallel_loop_dynamic
 #define KMP_API_NAME_GOMP_PARALLEL_LOOP_GUIDED GOMP_parallel_loop_guided
 #define KMP_API_NAME_GOMP_PARALLEL_LOOP_RUNTIME GOMP_parallel_loop_runtime
 #define KMP_API_NAME_GOMP_PARALLEL_LOOP_STATIC GOMP_parallel_loop_static

runtime/src/kmp_gsupport.cpp

if (ompt_enabled.enabled)
     OMPT_STORE_RETURN_ADDRESS(gtid);
 #endif
 
   __kmpc_end_taskgroup(&loc, gtid);
 
   return;
 }
 
-#ifndef KMP_DEBUG
-static
-#endif /* KMP_DEBUG */
-    kmp_int32
-    __kmp_gomp_to_omp_cancellation_kind(int gomp_kind) {
+static kmp_int32 __kmp_gomp_to_omp_cancellation_kind(int gomp_kind) {
   kmp_int32 cncl_kind = 0;
   switch (gomp_kind) {
   case 1:
     cncl_kind = cancel_parallel;
     break;
   case 2:
     cncl_kind = cancel_loop;
     break;
   case 4:
     cncl_kind = cancel_sections;
     break;
   case 8:
     cncl_kind = cancel_taskgroup;
     break;
   }
   return cncl_kind;
 }
 
+// Return true if cancellation should take place, false otherwise
 bool KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CANCELLATION_POINT)(int which) {
-  if (__kmp_omp_cancellation) {
-    KMP_FATAL(NoGompCancellation);
-  }
   int gtid = __kmp_get_gtid();
   MKLOC(loc, "GOMP_cancellation_point");
-  KA_TRACE(20, ("GOMP_cancellation_point: T#%d\n", gtid));
-
+  KA_TRACE(20, ("GOMP_cancellation_point: T#%d which:%d\n", gtid, which));
   kmp_int32 cncl_kind = __kmp_gomp_to_omp_cancellation_kind(which);
-
   return __kmpc_cancellationpoint(&loc, gtid, cncl_kind);
 }
 
-bool KMP_EXPAND_NAME(KMP_API_NAME_GOMP_BARRIER_CANCEL)(void) {
-  if (__kmp_omp_cancellation) {
-    KMP_FATAL(NoGompCancellation);
-  }
-  KMP_FATAL(NoGompCancellation);
-  int gtid = __kmp_get_gtid();
-  MKLOC(loc, "GOMP_barrier_cancel");
-  KA_TRACE(20, ("GOMP_barrier_cancel: T#%d\n", gtid));
-
-  return __kmpc_cancel_barrier(&loc, gtid);
-}
-
+// Return true if cancellation should take place, false otherwise
 bool KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CANCEL)(int which, bool do_cancel) {
-  if (__kmp_omp_cancellation) {
-    KMP_FATAL(NoGompCancellation);
-  } else {
-    return FALSE;
-  }
-
   int gtid = __kmp_get_gtid();
   MKLOC(loc, "GOMP_cancel");
-  KA_TRACE(20, ("GOMP_cancel: T#%d\n", gtid));
-
+  KA_TRACE(20, ("GOMP_cancel: T#%d which:%d do_cancel:%d\n", gtid, which,
+                (int)do_cancel));
   kmp_int32 cncl_kind = __kmp_gomp_to_omp_cancellation_kind(which);
 
   if (do_cancel == FALSE) {
-    return KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CANCELLATION_POINT)(which);
+    return __kmpc_cancellationpoint(&loc, gtid, cncl_kind);
   } else {
     return __kmpc_cancel(&loc, gtid, cncl_kind);
   }
 }
 
-bool KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SECTIONS_END_CANCEL)(void) {
-  if (__kmp_omp_cancellation) {
-    KMP_FATAL(NoGompCancellation);
-  }
+// Return true if cancellation should take place, false otherwise
+bool KMP_EXPAND_NAME(KMP_API_NAME_GOMP_BARRIER_CANCEL)(void) {
+  int gtid = __kmp_get_gtid();
+  KA_TRACE(20, ("GOMP_barrier_cancel: T#%d\n", gtid));
+  return __kmp_barrier_gomp_cancel(gtid);
+
+// Return true if cancellation should take place, false otherwise
+bool KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SECTIONS_END_CANCEL)(void) {
   int gtid = __kmp_get_gtid();
-  MKLOC(loc, "GOMP_sections_end_cancel");
   KA_TRACE(20, ("GOMP_sections_end_cancel: T#%d\n", gtid));
-
-  return __kmpc_cancel_barrier(&loc, gtid);
+  return __kmp_barrier_gomp_cancel(gtid);
 }
 
+// Return true if cancellation should take place, false otherwise
 bool KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_END_CANCEL)(void) {
-  if (__kmp_omp_cancellation) {
-    KMP_FATAL(NoGompCancellation);
-  }
   int gtid = __kmp_get_gtid();
-  MKLOC(loc, "GOMP_loop_end_cancel");
   KA_TRACE(20, ("GOMP_loop_end_cancel: T#%d\n", gtid));
-
-  return __kmpc_cancel_barrier(&loc, gtid);
+  return __kmp_barrier_gomp_cancel(gtid);
 }
 
 // All target functions are empty as of 2014-05-29
 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TARGET)(int device, void (*fn)(void *),
                                                const void *openmp_target,
                                                size_t mapnum, void **hostaddrs,
                                                size_t *sizes,
                                                unsigned char *kinds) {

runtime/src/kmp_wait_release.h

 }
 #endif
 
 /* Spin wait loop that first does pause, then yield, then sleep. A thread that
    calls __kmp_wait_*  must make certain that another thread calls __kmp_release
    to wake it back up to prevent deadlocks!
 
    NOTE: We may not belong to a team at this point.  */
-template <class C, int final_spin>
-static inline void
+template <class C, int final_spin, bool cancellable = false,
+          bool sleepable = true>
+static inline bool
 __kmp_wait_template(kmp_info_t *this_thr,
                     C *flag USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
 #if USE_ITT_BUILD && USE_ITT_NOTIFY
   volatile void *spin = flag->get();
 #endif
   kmp_uint32 spins;
   int th_gtid;
   int tasks_completed = FALSE;
   int oversubscribed;
 #if !KMP_USE_MONITOR
   kmp_uint64 poll_count;
   kmp_uint64 hibernate_goal;
 #else
   kmp_uint32 hibernate;
 #endif
 
   KMP_FSYNC_SPIN_INIT(spin, NULL);
   if (flag->done_check()) {
     KMP_FSYNC_SPIN_ACQUIRED(CCAST(void *, spin));
-    return;
+    return false;
   }
   th_gtid = this_thr->th.th_info.ds.ds_gtid;
+  if (cancellable) {
+    kmp_team_t *team = this_thr->th.th_team;
+    if (team && team->t.t_cancel_request == cancel_parallel)
+      return true;
+  }
 #if KMP_OS_UNIX
   if (final_spin)
     KMP_ATOMIC_ST_REL(&this_thr->th.th_blocking, true);
 #endif
   KA_TRACE(20,
            ("__kmp_wait_sleep: T#%d waiting for flag(%p)\n", th_gtid, flag));
 #if KMP_STATS_ENABLED
   stats_state_e thread_state = KMP_GET_THREAD_STATE();
#if KMP_STATS_ENABLED
     // Check if thread has been signalled to idle state
     // This indicates that the logical "join-barrier" has finished
     if (this_thr->th.th_stats->isIdle() &&
         KMP_GET_THREAD_STATE() == FORK_JOIN_BARRIER) {
       KMP_SET_THREAD_STATE(IDLE);
       KMP_PUSH_PARTITIONED_TIMER(OMP_idle);
     }
 #endif
+    // Check if the barrier surrounding this wait loop has been cancelled
+    if (cancellable) {
+      kmp_team_t *team = this_thr->th.th_team;
+      if (team && team->t.t_cancel_request == cancel_parallel)
+        break;
+    }
 
     // Don't suspend if KMP_BLOCKTIME is set to "infinite"
     if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME
 #if OMP_50_ENABLED
         && __kmp_pause_status != kmp_soft_paused
 #endif
         )
       continue;
 
     // Don't suspend if there is a likelihood of new tasks being spawned.
     if ((task_team != NULL) && TCR_4(task_team->tt.tt_found_tasks))
       continue;
 
 #if KMP_USE_MONITOR
     // If we have waited a bit more, fall asleep
     if (TCR_4(__kmp_global.g.g_time.dt.t_value) < hibernate)
       continue;
 #else
     if (KMP_BLOCKING(hibernate_goal, poll_count++))
       continue;
 #endif
+    // Don't suspend if wait loop designated non-sleepable
+    // in template parameters
+    if (!sleepable)
+      continue;
 
 #if OMP_50_ENABLED
     if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
         __kmp_pause_status != kmp_soft_paused)
       continue;
 #endif
 
     KF_TRACE(50, ("__kmp_wait_sleep: T#%d suspend time reached\n", th_gtid));
#if KMP_STATS_ENABLED
   }
 #endif
 
 #if KMP_OS_UNIX
   if (final_spin)
     KMP_ATOMIC_ST_REL(&this_thr->th.th_blocking, false);
 #endif
   KMP_FSYNC_SPIN_ACQUIRED(CCAST(void *, spin));
+  if (cancellable) {
+    kmp_team_t *team = this_thr->th.th_team;
+    if (team && team->t.t_cancel_request == cancel_parallel) {
+      if (tasks_completed) {
+        // undo the previous decrement of unfinished_threads so that the
+        // thread can decrement at the join barrier with no problem
+        kmp_task_team_t *task_team = this_thr->th.th_task_team;
+        std::atomic<kmp_int32> *unfinished_threads =
+            &(task_team->tt.tt_unfinished_threads);
+        KMP_ATOMIC_INC(unfinished_threads);
+      }
+      return true;
+    }
+  }
+  return false;
 }
 
 /* Release any threads specified as waiting on the flag by releasing the flag
    and resume the waiting thread if indicated by the sleep bit(s). A thread that
    calls __kmp_wait_template must call this function to wake up the potentially
    sleeping thread and prevent deadlocks!  */
 template <class C> static inline void __kmp_release_template(C *flag) {
 #ifdef KMP_DEBUG
void wait(kmp_info_t *this_thr,
             int final_spin USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
     if (final_spin)
       __kmp_wait_template<kmp_flag_64, TRUE>(
           this_thr, this USE_ITT_BUILD_ARG(itt_sync_obj));
     else
       __kmp_wait_template<kmp_flag_64, FALSE>(
           this_thr, this USE_ITT_BUILD_ARG(itt_sync_obj));
   }
+  bool wait_cancellable_nosleep(kmp_info_t *this_thr,
+                                int final_spin
+                                    USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
+    bool retval = false;
+    if (final_spin)
+      retval = __kmp_wait_template<kmp_flag_64, TRUE, true, false>(
+          this_thr, this USE_ITT_BUILD_ARG(itt_sync_obj));
+    else
+      retval = __kmp_wait_template<kmp_flag_64, FALSE, true, false>(
+          this_thr, this USE_ITT_BUILD_ARG(itt_sync_obj));
+    return retval;
+  }
   void release() { __kmp_release_template(this); }
   flag_type get_ptr_type() { return flag64; }
 };
 
 // Hierarchical 64-bit on-core barrier instantiation
 class kmp_flag_oncore : public kmp_flag_native<kmp_uint64> {
   kmp_uint64 checker;
   kmp_info_t *waiting_threads[1];