[OpenMP] Add OpenMP 5.0 nonmonotonic code

[jlpeyton]

Summary

This patch adds:

1. new omp_sched_monotonic flag to omp_sched_t which is handled within the runtime
2. parsing of monotonic/nonmonotonic in OMP_SCHEDULE
3. tests for the monotonic flag and envirable parsing
4. logic to force monotonic when hierarchical scheduling is used

[AndreyChurbanov]

LGTM

openmp/trunk/runtime/src/include/50/omp.h.var

 #       define __KAI_KMPC_CONVENTION
 #       ifndef __KMP_IMP
 #           define __KMP_IMP
 #       endif
 #   endif
 
     /* schedule kind constants */
     typedef enum omp_sched_t {
-  omp_sched_static  = 1,
-  omp_sched_dynamic = 2,
-  omp_sched_guided  = 3,
-  omp_sched_auto    = 4
+        omp_sched_static  = 1,
+        omp_sched_dynamic = 2,
+        omp_sched_guided  = 3,
+        omp_sched_auto    = 4,
+        omp_sched_monotonic = 0x80000000
     } omp_sched_t;
 
     /* set API functions */
     extern void   __KAI_KMPC_CONVENTION  omp_set_num_threads (int);
     extern void   __KAI_KMPC_CONVENTION  omp_set_dynamic     (int);
     extern void   __KAI_KMPC_CONVENTION  omp_set_nested      (int);
     extern void   __KAI_KMPC_CONVENTION  omp_set_max_active_levels (int);
     extern void   __KAI_KMPC_CONVENTION  omp_set_schedule          (omp_sched_t, int);

openmp/trunk/runtime/src/include/50/omp_lib.h.var

!
       integer(kind=omp_sched_kind)omp_sched_static
       parameter(omp_sched_static=1)
       integer(kind=omp_sched_kind)omp_sched_dynamic
       parameter(omp_sched_dynamic=2)
       integer(kind=omp_sched_kind)omp_sched_guided
       parameter(omp_sched_guided=3)
       integer(kind=omp_sched_kind)omp_sched_auto
       parameter(omp_sched_auto=4)
+      integer(kind=omp_sched_kind)omp_sched_monotonic
+      parameter(omp_sched_monotonic=Z'80000000')
 
       integer(kind=omp_proc_bind_kind)omp_proc_bind_false
       parameter(omp_proc_bind_false=0)
       integer(kind=omp_proc_bind_kind)omp_proc_bind_true
       parameter(omp_proc_bind_true=1)
       integer(kind=omp_proc_bind_kind)omp_proc_bind_master
       parameter(omp_proc_bind_master=2)
       integer(kind=omp_proc_bind_kind)omp_proc_bind_close

openmp/trunk/runtime/src/include/50/omp_lib.f.var

!dec$ fixedformlinesize:132
         integer (kind=omp_integer_kind), parameter :: kmp_version_build = @LIBOMP_VERSION_BUILD@
         character(*), parameter :: kmp_build_date    = '@LIBOMP_BUILD_DATE@'
         integer (kind=omp_integer_kind), parameter :: openmp_version    = @LIBOMP_OMP_YEAR_MONTH@
 
         integer(kind=omp_sched_kind), parameter :: omp_sched_static  = 1
         integer(kind=omp_sched_kind), parameter :: omp_sched_dynamic = 2
         integer(kind=omp_sched_kind), parameter :: omp_sched_guided  = 3
         integer(kind=omp_sched_kind), parameter :: omp_sched_auto    = 4
+        integer(kind=omp_sched_kind), parameter :: omp_sched_monotonic = Z'80000000'
 
         integer (kind=omp_proc_bind_kind), parameter :: omp_proc_bind_false = 0
         integer (kind=omp_proc_bind_kind), parameter :: omp_proc_bind_true = 1
         integer (kind=omp_proc_bind_kind), parameter :: omp_proc_bind_master = 2
         integer (kind=omp_proc_bind_kind), parameter :: omp_proc_bind_close = 3
         integer (kind=omp_proc_bind_kind), parameter :: omp_proc_bind_spread = 4
 
         integer (kind=kmp_cancel_kind), parameter :: kmp_cancel_parallel = 1

openmp/trunk/runtime/src/include/50/omp_lib.f90.var

!
         integer (kind=omp_integer_kind), parameter :: kmp_version_build = @LIBOMP_VERSION_BUILD@
         character(*)               kmp_build_date
         parameter( kmp_build_date = '@LIBOMP_BUILD_DATE@' )
 
         integer(kind=omp_sched_kind), parameter :: omp_sched_static  = 1
         integer(kind=omp_sched_kind), parameter :: omp_sched_dynamic = 2
         integer(kind=omp_sched_kind), parameter :: omp_sched_guided  = 3
         integer(kind=omp_sched_kind), parameter :: omp_sched_auto    = 4
-
+        integer(kind=omp_sched_kind), parameter :: omp_sched_monotonic = Z'80000000'
 
         integer (kind=omp_proc_bind_kind), parameter :: omp_proc_bind_false = 0
         integer (kind=omp_proc_bind_kind), parameter :: omp_proc_bind_true = 1
         integer (kind=omp_proc_bind_kind), parameter :: omp_proc_bind_master = 2
         integer (kind=omp_proc_bind_kind), parameter :: omp_proc_bind_close = 3
         integer (kind=omp_proc_bind_kind), parameter :: omp_proc_bind_spread = 4
 
         integer (kind=kmp_cancel_kind), parameter :: kmp_cancel_parallel = 1

openmp/trunk/runtime/src/kmp.h

typedef enum kmp_sched {
   kmp_sched_auto = 4, // mapped to kmp_sch_auto                     (38)
   kmp_sched_upper_std = 5, // upper bound for standard schedules
   kmp_sched_lower_ext = 100, // lower bound of Intel extension schedules
   kmp_sched_trapezoidal = 101, // mapped to kmp_sch_trapezoidal (39)
 #if KMP_STATIC_STEAL_ENABLED
   kmp_sched_static_steal = 102, // mapped to kmp_sch_static_steal (44)
 #endif
   kmp_sched_upper,
-  kmp_sched_default = kmp_sched_static // default scheduling
+  kmp_sched_default = kmp_sched_static, // default scheduling
+  kmp_sched_monotonic = 0x80000000
 } kmp_sched_t;
 #endif
 
 /*!
  @ingroup WORK_SHARING
  * Describes the loop schedule to be used for a parallel for loop.
  */
 enum sched_type : kmp_int32 {

 
 #define SCHEDULE_WITHOUT_MODIFIERS(s)                                          \
   (enum sched_type)(                                                           \
       (s) & ~(kmp_sch_modifier_nonmonotonic | kmp_sch_modifier_monotonic))
 #define SCHEDULE_HAS_MONOTONIC(s) (((s)&kmp_sch_modifier_monotonic) != 0)
 #define SCHEDULE_HAS_NONMONOTONIC(s) (((s)&kmp_sch_modifier_nonmonotonic) != 0)
 #define SCHEDULE_HAS_NO_MODIFIERS(s)                                           \
   (((s) & (kmp_sch_modifier_nonmonotonic | kmp_sch_modifier_monotonic)) == 0)
+#define SCHEDULE_GET_MODIFIERS(s)                                              \
+  ((enum sched_type)(                                                          \
+      (s) & (kmp_sch_modifier_nonmonotonic | kmp_sch_modifier_monotonic)))
+#define SCHEDULE_SET_MODIFIERS(s, m)                                           \
+  (s = (enum sched_type)((kmp_int32)s | (kmp_int32)m))
 #else
 /* By doing this we hope to avoid multiple tests on OMP_45_ENABLED. Compilers
    can now eliminate tests on compile time constants and dead code that results
    from them, so we can leave code guarded by such an if in place.  */
 #define SCHEDULE_WITHOUT_MODIFIERS(s) (s)
 #define SCHEDULE_HAS_MONOTONIC(s) false
 #define SCHEDULE_HAS_NONMONOTONIC(s) false
 #define SCHEDULE_HAS_NO_MODIFIERS(s) true
+#define SCHEDULE_GET_MODIFIERS(s) ((enum sched_type)0)
+#define SCHEDULE_SET_MODIFIERS(s, m) /* Nothing */
 #endif
+#define SCHEDULE_NONMONOTONIC 0
+#define SCHEDULE_MONOTONIC 1
 
   kmp_sch_default = kmp_sch_static /**< default scheduling algorithm */
 };
 
+// Apply modifiers on internal kind to standard kind
+static inline void
+__kmp_sched_apply_mods_stdkind(kmp_sched_t *kind,
+                               enum sched_type internal_kind) {
+#if OMP_50_ENABLED
+  if (SCHEDULE_HAS_MONOTONIC(internal_kind)) {
+    *kind = (kmp_sched_t)((int)*kind | (int)kmp_sched_monotonic);
+  }
+#endif
+}
+
+// Apply modifiers on standard kind to internal kind
+static inline void
+__kmp_sched_apply_mods_intkind(kmp_sched_t kind,
+                               enum sched_type *internal_kind) {
+#if OMP_50_ENABLED
+  if ((int)kind & (int)kmp_sched_monotonic) {
+    *internal_kind = (enum sched_type)((int)*internal_kind |
+                                       (int)kmp_sch_modifier_monotonic);
+  }
+#endif
+}
+
+// Get standard schedule without modifiers
+static inline kmp_sched_t __kmp_sched_without_mods(kmp_sched_t kind) {
+#if OMP_50_ENABLED
+  return (kmp_sched_t)((int)kind & ~((int)kmp_sched_monotonic));
+#else
+  return kind;
+#endif
+}
+
 /* Type to keep runtime schedule set via OMP_SCHEDULE or omp_set_schedule() */
 typedef union kmp_r_sched {
   struct {
     enum sched_type r_sched_type;
     int chunk;
   };
   kmp_int64 sched;
 } kmp_r_sched_t;

openmp/trunk/runtime/src/kmp_dispatch.cpp

void __kmp_dispatch_dxo_error(int *gtid_ref, int *cid_ref, ident_t *loc_ref) {
   if (__kmp_env_consistency_check) {
     th = __kmp_threads[*gtid_ref];
     if (th->th.th_dispatch->th_dispatch_pr_current->pushed_ws != ct_none) {
       __kmp_pop_sync(*gtid_ref, ct_ordered_in_pdo, loc_ref);
     }
   }
 }
 
+// Returns either SCHEDULE_MONOTONIC or SCHEDULE_NONMONOTONIC
+static inline int __kmp_get_monotonicity(enum sched_type schedule,
+                                         bool use_hier = false) {
+  // Pick up the nonmonotonic/monotonic bits from the scheduling type
+  int monotonicity;
+  // default to monotonic
+  monotonicity = SCHEDULE_MONOTONIC;
+  if (SCHEDULE_HAS_NONMONOTONIC(schedule))
+    monotonicity = SCHEDULE_NONMONOTONIC;
+  else if (SCHEDULE_HAS_MONOTONIC(schedule))
+    monotonicity = SCHEDULE_MONOTONIC;
+  return monotonicity;
+}
+
 // Initialize a dispatch_private_info_template<T> buffer for a particular
 // type of schedule,chunk.  The loop description is found in lb (lower bound),
 // ub (upper bound), and st (stride).  nproc is the number of threads relevant
 // to the scheduling (often the number of threads in a team, but not always if
 // hierarchical scheduling is used).  tid is the id of the thread calling
 // the function within the group of nproc threads.  It will have a value
 // between 0 and nproc - 1.  This is often just the thread id within a team, but
 // is not necessarily the case when using hierarchical scheduling.
#endif
                                    T nproc, T tid) {
   typedef typename traits_t<T>::unsigned_t UT;
   typedef typename traits_t<T>::floating_t DBL;
 
   int active;
   T tc;
   kmp_info_t *th;
   kmp_team_t *team;
+  int monotonicity;
+  bool use_hier;
 
 #ifdef KMP_DEBUG
   typedef typename traits_t<T>::signed_t ST;
   {
     char *buff;
     // create format specifiers before the debug output
     buff = __kmp_str_format("__kmp_dispatch_init_algorithm: T#%%d called "
                             "pr:%%p lb:%%%s ub:%%%s st:%%%s "
#if USE_ITT_BUILD
   int itt_need_metadata_reporting = __itt_metadata_add_ptr &&
                                     __kmp_forkjoin_frames_mode == 3 &&
                                     KMP_MASTER_GTID(gtid) &&
 #if OMP_40_ENABLED
                                     th->th.th_teams_microtask == NULL &&
 #endif
                                     team->t.t_active_level == 1;
 #endif
-#if (KMP_STATIC_STEAL_ENABLED)
-  if (SCHEDULE_HAS_NONMONOTONIC(schedule))
-    // AC: we now have only one implementation of stealing, so use it
-    schedule = kmp_sch_static_steal;
-  else
+
+#if KMP_USE_HIER_SCHED
+  use_hier = pr->flags.use_hier;
+#else
+  use_hier = false;
 #endif
-    schedule = SCHEDULE_WITHOUT_MODIFIERS(schedule);
+
+  /* Pick up the nonmonotonic/monotonic bits from the scheduling type */
+  monotonicity = __kmp_get_monotonicity(schedule, use_hier);
 
   /* Pick up the nomerge/ordered bits from the scheduling type */
   if ((schedule >= kmp_nm_lower) && (schedule < kmp_nm_upper)) {
     pr->flags.nomerge = TRUE;
     schedule =
         (enum sched_type)(((int)schedule) - (kmp_nm_lower - kmp_sch_lower));
   } else {
     pr->flags.nomerge = FALSE;
   }
   pr->type_size = traits_t<T>::type_size; // remember the size of variables
   if (kmp_ord_lower & schedule) {
     pr->flags.ordered = TRUE;
     schedule =
         (enum sched_type)(((int)schedule) - (kmp_ord_lower - kmp_sch_lower));
   } else {
     pr->flags.ordered = FALSE;
   }
+  // Ordered overrides nonmonotonic
+  if (pr->flags.ordered) {
+    monotonicity = SCHEDULE_MONOTONIC;
+  }
 
   if (schedule == kmp_sch_static) {
     schedule = __kmp_static;
   } else {
     if (schedule == kmp_sch_runtime) {
       // Use the scheduling specified by OMP_SCHEDULE (or __kmp_sch_default if
       // not specified)
       schedule = team->t.t_sched.r_sched_type;
+      monotonicity = __kmp_get_monotonicity(schedule, use_hier);
+      schedule = SCHEDULE_WITHOUT_MODIFIERS(schedule);
       // Detail the schedule if needed (global controls are differentiated
       // appropriately)
       if (schedule == kmp_sch_guided_chunked) {
         schedule = __kmp_guided;
       } else if (schedule == kmp_sch_static) {
         schedule = __kmp_static;
       }
       // Use the chunk size specified by OMP_SCHEDULE (or default if not
#ifdef KMP_DEBUG
             "__kmp_dispatch_init_algorithm: kmp_sch_auto: T#%%d new: "
             "schedule:%%d chunk:%%%s\n",
             traits_t<ST>::spec);
         KD_TRACE(10, (buff, gtid, schedule, chunk));
         __kmp_str_free(&buff);
       }
 #endif
     }
-
+#if KMP_STATIC_STEAL_ENABLED
+    // map nonmonotonic:dynamic to static steal
+    if (schedule == kmp_sch_dynamic_chunked) {
+      if (monotonicity == SCHEDULE_NONMONOTONIC)
+        schedule = kmp_sch_static_steal;
+    }
+#endif
     /* guided analytical not safe for too many threads */
     if (schedule == kmp_sch_guided_analytical_chunked && nproc > 1 << 20) {
       schedule = kmp_sch_guided_iterative_chunked;
       KMP_WARNING(DispatchManyThreads);
     }
 #if OMP_45_ENABLED
     if (schedule == kmp_sch_runtime_simd) {
       // compiler provides simd_width in the chunk parameter
       schedule = team->t.t_sched.r_sched_type;
+      monotonicity = __kmp_get_monotonicity(schedule, use_hier);
+      schedule = SCHEDULE_WITHOUT_MODIFIERS(schedule);
       // Detail the schedule if needed (global controls are differentiated
       // appropriately)
       if (schedule == kmp_sch_static || schedule == kmp_sch_auto ||
           schedule == __kmp_static) {
         schedule = kmp_sch_static_balanced_chunked;
       } else {
         if (schedule == kmp_sch_guided_chunked || schedule == __kmp_guided) {
           schedule = kmp_sch_guided_simd;
         }
         chunk = team->t.t_sched.chunk * chunk;
       }
 #if USE_ITT_BUILD
       if (cur_chunk)
         *cur_chunk = chunk;
 #endif
 #ifdef KMP_DEBUG
       {
         char *buff;
         // create format specifiers before the debug output
-        buff = __kmp_str_format("__kmp_dispatch_init: T#%%d new: schedule:%%d"
-                                " chunk:%%%s\n",
-                                traits_t<ST>::spec);
+        buff = __kmp_str_format(
+            "__kmp_dispatch_init_algorithm: T#%%d new: schedule:%%d"
         KD_TRACE(10, (buff, gtid, schedule, chunk));
         __kmp_str_free(&buff);
       }
 #endif
     }
 #endif // OMP_45_ENABLED
     pr->u.p.parm1 = chunk;
   }
if (nproc > 1 && ntc >= nproc) {
       small_chunk = ntc / nproc;
       extras = ntc % nproc;
 
       init = id * small_chunk + (id < extras ? id : extras);
       pr->u.p.count = init;
       pr->u.p.ub = init + small_chunk + (id < extras ? 1 : 0);
 
       pr->u.p.parm2 = lb;
-      // pr->pfields.parm3 = 0; // it's not used in static_steal
+      // parm3 is the number of times to attempt stealing which is
+      // proportional to the number of chunks per thread up until
+      // the maximum value of nproc.
+      pr->u.p.parm3 = KMP_MIN(small_chunk + extras, nproc);
       pr->u.p.parm4 = (id + 1) % nproc; // remember neighbour tid
       pr->u.p.st = st;
       if (traits_t<T>::type_size > 4) {
         // AC: TODO: check if 16-byte CAS available and use it to
         // improve performance (probably wait for explicit request
         // before spending time on this).
         // For now use dynamically allocated per-thread lock,
         // free memory in __kmp_dispatch_next when status==0.
if (traits_t<T>::type_size > 4) {
         init = (pr->u.p.count)++;
         status = (init < (UT)pr->u.p.ub);
         __kmp_release_lock(lck, gtid);
       } else {
         status = 0; // no own chunks
       }
       if (!status) { // try to steal
         kmp_info_t **other_threads = team->t.t_threads;
-        int while_limit = nproc; // nproc attempts to find a victim
+        int while_limit = pr->u.p.parm3;
         int while_index = 0;
         // TODO: algorithm of searching for a victim
         // should be cleaned up and measured
         while ((!status) && (while_limit != ++while_index)) {
           T remaining;
           T victimIdx = pr->u.p.parm4;
           T oldVictimIdx = victimIdx ? victimIdx - 1 : nproc - 1;
           dispatch_private_info_template<T> *victim =
if (traits_t<T>::type_size > 4) {
         }
         vnew = vold;
         init = vnew.p.count;
         status = (init < (UT)vnew.p.ub);
       }
 
       if (!status) {
         kmp_info_t **other_threads = team->t.t_threads;
-        int while_limit = nproc; // nproc attempts to find a victim
+        int while_limit = pr->u.p.parm3;
         int while_index = 0;
 
         // TODO: algorithm of searching for a victim
         // should be cleaned up and measured
         while ((!status) && (while_limit != ++while_index)) {
           union_i4 vold, vnew;
           kmp_int32 remaining;
           T victimIdx = pr->u.p.parm4;

openmp/trunk/runtime/src/kmp_dispatch_hier.h

for (int i = 0; i < n; ++i) {
         deallocate();
         return;
       }
       info[i].length = max;
       layers[i] = (kmp_hier_top_unit_t<T> *)__kmp_allocate(
           sizeof(kmp_hier_top_unit_t<T>) * max);
       for (int j = 0; j < max; ++j) {
         layers[i][j].active = 0;
+        layers[i][j].hier_pr.flags.use_hier = TRUE;
       }
     }
     valid = true;
   }
   // loc - source file location
   // gtid - global thread identifier
   // pr - this thread's private dispatch buffer (corresponding with gtid)
   // p_last (return value) - pointer to flag indicating this set of iterations
#if OMP_50_ENABLED
   __kmp_resume_if_soft_paused();
 #endif
 
   th = __kmp_threads[gtid];
   team = th->th.th_team;
   active = !team->t.t_serialized;
   th->th.th_ident = loc;
   num_hw_threads = __kmp_hier_max_units[kmp_hier_layer_e::LAYER_THREAD + 1];
+  KMP_DEBUG_ASSERT(th->th.th_dispatch ==
+                   &th->th.th_team->t.t_dispatch[th->th.th_info.ds.ds_tid]);
+  my_buffer_index = th->th.th_dispatch->th_disp_index;
+  pr = reinterpret_cast<dispatch_private_info_template<T> *>(
+      &th->th.th_dispatch
+           ->th_disp_buffer[my_buffer_index % __kmp_dispatch_num_buffers]);
+  sh = reinterpret_cast<dispatch_shared_info_template<T> volatile *>(
+      &team->t.t_disp_buffer[my_buffer_index % __kmp_dispatch_num_buffers]);
   if (!active) {
     KD_TRACE(10, ("__kmp_dispatch_init_hierarchy: T#%d not active parallel. "
                   "Using normal dispatch functions.\n",
                   gtid));
-    pr = reinterpret_cast<dispatch_private_info_template<T> *>(
-        th->th.th_dispatch->th_disp_buffer);
     KMP_DEBUG_ASSERT(pr);
     pr->flags.use_hier = FALSE;
     pr->flags.contains_last = FALSE;
     return;
   }
-  KMP_DEBUG_ASSERT(th->th.th_dispatch ==
-                   &th->th.th_team->t.t_dispatch[th->th.th_info.ds.ds_tid]);
-
-  my_buffer_index = th->th.th_dispatch->th_disp_index;
-  pr = reinterpret_cast<dispatch_private_info_template<T> *>(
-      &th->th.th_dispatch
-           ->th_disp_buffer[my_buffer_index % __kmp_dispatch_num_buffers]);
-  sh = reinterpret_cast<dispatch_shared_info_template<T> volatile *>(
-      &team->t.t_disp_buffer[my_buffer_index % __kmp_dispatch_num_buffers]);
   KMP_DEBUG_ASSERT(pr);
   KMP_DEBUG_ASSERT(sh);
   pr->flags.use_hier = TRUE;
   pr->u.p.tc = 0;
   // Have master allocate the hierarchy
   if (__kmp_tid_from_gtid(gtid) == 0) {
     KD_TRACE(10, ("__kmp_dispatch_init_hierarchy: T#%d pr:%p sh:%p allocating "
                   "hierarchy\n",

openmp/trunk/runtime/src/kmp_runtime.cpp

int __kmp_get_max_active_levels(int gtid) {
   KMP_DEBUG_ASSERT(thread->th.th_current_task);
   KF_TRACE(10, ("__kmp_get_max_active_levels: thread %d, curtask=%p, "
                 "curtask_maxaclevel=%d\n",
                 gtid, thread->th.th_current_task,
                 thread->th.th_current_task->td_icvs.max_active_levels));
   return thread->th.th_current_task->td_icvs.max_active_levels;
 }
 
+KMP_BUILD_ASSERT(sizeof(kmp_sched_t) == sizeof(int));
+KMP_BUILD_ASSERT(sizeof(enum sched_type) == sizeof(int));
+
 /* Changes def_sched_var ICV values (run-time schedule kind and chunk) */
 void __kmp_set_schedule(int gtid, kmp_sched_t kind, int chunk) {
   kmp_info_t *thread;
+  kmp_sched_t orig_kind;
   //    kmp_team_t *team;
 
   KF_TRACE(10, ("__kmp_set_schedule: new schedule for thread %d = (%d, %d)\n",
                 gtid, (int)kind, chunk));
   KMP_DEBUG_ASSERT(__kmp_init_serial);
 
   // Check if the kind parameter is valid, correct if needed.
   // Valid parameters should fit in one of two intervals - standard or extended:
   //       <lower>, <valid>, <upper_std>, <lower_ext>, <valid>, <upper>
   // 2008-01-25: 0,  1 - 4,       5,         100,     101 - 102, 103
+  orig_kind = kind;
+  kind = __kmp_sched_without_mods(kind);
+
   if (kind <= kmp_sched_lower || kind >= kmp_sched_upper ||
       (kind <= kmp_sched_lower_ext && kind >= kmp_sched_upper_std)) {
     // TODO: Hint needs attention in case we change the default schedule.
     __kmp_msg(kmp_ms_warning, KMP_MSG(ScheduleKindOutOfRange, kind),
               KMP_HNT(DefaultScheduleKindUsed, "static, no chunk"),
               __kmp_msg_null);
     kind = kmp_sched_default;
     chunk = 0; // ignore chunk value in case of bad kind
if (kind < kmp_sched_upper_std) {
     }
   } else {
     //    __kmp_sch_map[ kind - kmp_sched_lower_ext + kmp_sched_upper_std -
     //    kmp_sched_lower - 2 ];
     thread->th.th_current_task->td_icvs.sched.r_sched_type =
         __kmp_sch_map[kind - kmp_sched_lower_ext + kmp_sched_upper_std -
                       kmp_sched_lower - 2];
   }
+  __kmp_sched_apply_mods_intkind(
+      orig_kind, &(thread->th.th_current_task->td_icvs.sched.r_sched_type));
   if (kind == kmp_sched_auto || chunk < 1) {
     // ignore parameter chunk for schedule auto
     thread->th.th_current_task->td_icvs.sched.chunk = KMP_DEFAULT_CHUNK;
   } else {
     thread->th.th_current_task->td_icvs.sched.chunk = chunk;
   }
 }
 
 /* Gets def_sched_var ICV values */
 void __kmp_get_schedule(int gtid, kmp_sched_t *kind, int *chunk) {
   kmp_info_t *thread;
   enum sched_type th_type;
 
   KF_TRACE(10, ("__kmp_get_schedule: thread %d\n", gtid));
   KMP_DEBUG_ASSERT(__kmp_init_serial);
 
   thread = __kmp_threads[gtid];
 
   th_type = thread->th.th_current_task->td_icvs.sched.r_sched_type;
-
-  switch (th_type) {
+  switch (SCHEDULE_WITHOUT_MODIFIERS(th_type)) {
   case kmp_sch_static:
   case kmp_sch_static_greedy:
   case kmp_sch_static_balanced:
     *kind = kmp_sched_static;
+    __kmp_sched_apply_mods_stdkind(kind, th_type);
     *chunk = 0; // chunk was not set, try to show this fact via zero value
     return;
   case kmp_sch_static_chunked:
     *kind = kmp_sched_static;
     break;
   case kmp_sch_dynamic_chunked:
     *kind = kmp_sched_dynamic;
     break;
#if KMP_STATIC_STEAL_ENABLED
   case kmp_sch_static_steal:
     *kind = kmp_sched_static_steal;
     break;
 #endif
   default:
     KMP_FATAL(UnknownSchedulingType, th_type);
   }
 
+  __kmp_sched_apply_mods_stdkind(kind, th_type);
   *chunk = thread->th.th_current_task->td_icvs.sched.chunk;
 }
 
 int __kmp_get_ancestor_thread_num(int gtid, int level) {
 
   int ii, dd;
   kmp_team_t *team;
   kmp_info_t *thr;
kmp_r_sched_t __kmp_get_schedule_global() {
   // independently. So one can get the updated schedule here.
 
   kmp_r_sched_t r_sched;
 
   // create schedule from 4 globals: __kmp_sched, __kmp_chunk, __kmp_static,
   // __kmp_guided. __kmp_sched should keep original value, so that user can set
   // KMP_SCHEDULE multiple times, and thus have different run-time schedules in
   // different roots (even in OMP 2.5)
-  if (__kmp_sched == kmp_sch_static) {
+  enum sched_type s = SCHEDULE_WITHOUT_MODIFIERS(__kmp_sched);
+#if OMP_45_ENABLED
+  enum sched_type sched_modifiers = SCHEDULE_GET_MODIFIERS(__kmp_sched);
+#endif
+  if (s == kmp_sch_static) {
     // replace STATIC with more detailed schedule (balanced or greedy)
     r_sched.r_sched_type = __kmp_static;
-  } else if (__kmp_sched == kmp_sch_guided_chunked) {
+  } else if (s == kmp_sch_guided_chunked) {
     // replace GUIDED with more detailed schedule (iterative or analytical)
     r_sched.r_sched_type = __kmp_guided;
   } else { // (STATIC_CHUNKED), or (DYNAMIC_CHUNKED), or other
     r_sched.r_sched_type = __kmp_sched;
   }
+#if OMP_45_ENABLED
+  SCHEDULE_SET_MODIFIERS(r_sched.r_sched_type, sched_modifiers);
+#endif
 
   if (__kmp_chunk < KMP_DEFAULT_CHUNK) {
     // __kmp_chunk may be wrong here (if it was not ever set)
     r_sched.chunk = KMP_DEFAULT_CHUNK;
   } else {
     r_sched.chunk = __kmp_chunk;
   }
 

openmp/trunk/runtime/src/kmp_settings.cpp

 static inline void __kmp_omp_schedule_restore() {
 #if KMP_USE_HIER_SCHED
   __kmp_hier_scheds.deallocate();
 #endif
   __kmp_chunk = 0;
   __kmp_sched = kmp_sch_default;
 }
 
+// if parse_hier = true:
+//    Parse [HW,][modifier:]kind[,chunk]
+// else:
+//    Parse [modifier:]kind[,chunk]
 static const char *__kmp_parse_single_omp_schedule(const char *name,
                                                    const char *value,
                                                    bool parse_hier = false) {
   /* get the specified scheduling style */
   const char *ptr = value;
-  const char *comma = strchr(ptr, ',');
   const char *delim;
   int chunk = 0;
   enum sched_type sched = kmp_sch_default;
   if (*ptr == '\0')
     return NULL;
+  delim = ptr;
+  while (*delim != ',' && *delim != ':' && *delim != '\0')
+    delim++;
 #if KMP_USE_HIER_SCHED
   kmp_hier_layer_e layer = kmp_hier_layer_e::LAYER_THREAD;
   if (parse_hier) {
+    if (*delim == ',') {
       if (!__kmp_strcasecmp_with_sentinel("L1", ptr, ',')) {
         layer = kmp_hier_layer_e::LAYER_L1;
       } else if (!__kmp_strcasecmp_with_sentinel("L2", ptr, ',')) {
         layer = kmp_hier_layer_e::LAYER_L2;
       } else if (!__kmp_strcasecmp_with_sentinel("L3", ptr, ',')) {
         layer = kmp_hier_layer_e::LAYER_L3;
       } else if (!__kmp_strcasecmp_with_sentinel("NUMA", ptr, ',')) {
         layer = kmp_hier_layer_e::LAYER_NUMA;
       }
-    if (layer != kmp_hier_layer_e::LAYER_THREAD && !comma) {
+    }
+    if (layer != kmp_hier_layer_e::LAYER_THREAD && *delim != ',') {
       // If there is no comma after the layer, then this schedule is invalid
       KMP_WARNING(StgInvalidValue, name, value);
       __kmp_omp_schedule_restore();
       return NULL;
     } else if (layer != kmp_hier_layer_e::LAYER_THREAD) {
-      ptr = ++comma;
-      comma = strchr(ptr, ',');
+      ptr = ++delim;
+      while (*delim != ',' && *delim != ':' && *delim != '\0')
+        delim++;
     }
   }
-  delim = ptr;
+#endif // KMP_USE_HIER_SCHED
+#if OMP_45_ENABLED
+  // Read in schedule modifier if specified
+  enum sched_type sched_modifier = (enum sched_type)0;
+  if (*delim == ':') {
+    if (!__kmp_strcasecmp_with_sentinel("monotonic", ptr, *delim)) {
+      sched_modifier = sched_type::kmp_sch_modifier_monotonic;
+      ptr = ++delim;
       while (*delim != ',' && *delim != ':' && *delim != '\0')
         delim++;
-#else // KMP_USE_HIER_SCHED
-  delim = ptr;
-  while (*delim != ',' && *delim != '\0')
+    } else if (!__kmp_strcasecmp_with_sentinel("nonmonotonic", ptr, *delim)) {
+      sched_modifier = sched_type::kmp_sch_modifier_nonmonotonic;
+      ptr = ++delim;
+      while (*delim != ',' && *delim != ':' && *delim != '\0')
         delim++;
-#endif // KMP_USE_HIER_SCHED
-  if (!__kmp_strcasecmp_with_sentinel("dynamic", ptr, *delim)) /* DYNAMIC */
+    } else if (!parse_hier) {
+      // If there is no proper schedule modifier, then this schedule is invalid
+      KMP_WARNING(StgInvalidValue, name, value);
+      __kmp_omp_schedule_restore();
+      return NULL;
+    }
+  }
+#endif
+  // Read in schedule kind (required)
+  if (!__kmp_strcasecmp_with_sentinel("dynamic", ptr, *delim))
     sched = kmp_sch_dynamic_chunked;
-  else if (!__kmp_strcasecmp_with_sentinel("guided", ptr, *delim)) /* GUIDED */
+  else if (!__kmp_strcasecmp_with_sentinel("guided", ptr, *delim))
     sched = kmp_sch_guided_chunked;
-  // AC: TODO: add AUTO schedule, and probably remove TRAPEZOIDAL (OMP 3.0 does
-  // not allow it)
-  else if (!__kmp_strcasecmp_with_sentinel("auto", ptr, *delim)) { /* AUTO */
+  // AC: TODO: probably remove TRAPEZOIDAL (OMP 3.0 does not allow it)
+  else if (!__kmp_strcasecmp_with_sentinel("auto", ptr, *delim))
     sched = kmp_sch_auto;
-    if (comma) {
-      __kmp_msg(kmp_ms_warning, KMP_MSG(IgnoreChunk, name, comma),
-                __kmp_msg_null);
-      comma = NULL;
-    }
-  } else if (!__kmp_strcasecmp_with_sentinel("trapezoidal", ptr,
-                                             *delim)) /* TRAPEZOIDAL */
+  else if (!__kmp_strcasecmp_with_sentinel("trapezoidal", ptr, *delim))
     sched = kmp_sch_trapezoidal;
-  else if (!__kmp_strcasecmp_with_sentinel("static", ptr, *delim)) /* STATIC */
+  else if (!__kmp_strcasecmp_with_sentinel("static", ptr, *delim))
     sched = kmp_sch_static;
 #if KMP_STATIC_STEAL_ENABLED
   else if (!__kmp_strcasecmp_with_sentinel("static_steal", ptr, *delim))
     sched = kmp_sch_static_steal;
 #endif
   else {
+    // If there is no proper schedule kind, then this schedule is invalid
     KMP_WARNING(StgInvalidValue, name, value);
     __kmp_omp_schedule_restore();
     return NULL;
   }
-  if (ptr && comma && *comma == *delim) {
-    ptr = comma + 1;
-    SKIP_DIGITS(ptr);
 
+  // Read in schedule chunk size if specified
+  if (*delim == ',') {
+    ptr = delim + 1;
+    SKIP_WS(ptr);
+    if (!isdigit(*ptr)) {
+      // If there is no chunk after comma, then this schedule is invalid
+      KMP_WARNING(StgInvalidValue, name, value);
+      __kmp_omp_schedule_restore();
+      return NULL;
+    }
+    SKIP_DIGITS(ptr);
+    // auto schedule should not specify chunk size
+    if (sched == kmp_sch_auto) {
+      __kmp_msg(kmp_ms_warning, KMP_MSG(IgnoreChunk, name, delim),
+                __kmp_msg_null);
+    } else {
       if (sched == kmp_sch_static)
         sched = kmp_sch_static_chunked;
-    ++comma;
-    chunk = __kmp_str_to_int(comma, *ptr);
+      chunk = __kmp_str_to_int(delim + 1, *ptr);
       if (chunk < 1) {
         chunk = KMP_DEFAULT_CHUNK;
-      __kmp_msg(kmp_ms_warning, KMP_MSG(InvalidChunk, name, comma),
+        __kmp_msg(kmp_ms_warning, KMP_MSG(InvalidChunk, name, delim),
                   __kmp_msg_null);
         KMP_INFORM(Using_int_Value, name, __kmp_chunk);
         // AC: next block commented out until KMP_DEFAULT_CHUNK != KMP_MIN_CHUNK
         // (to improve code coverage :)
-      //     The default chunk size is 1 according to standard, thus making
-      //     KMP_MIN_CHUNK not 1 we would introduce mess:
-      //     wrong chunk becomes 1, but it will be impossible to explicitely set
-      //     1, because it becomes KMP_MIN_CHUNK...
-      //                } else if ( chunk < KMP_MIN_CHUNK ) {
-      //                    chunk = KMP_MIN_CHUNK;
+        // The default chunk size is 1 according to standard, thus making
+        // KMP_MIN_CHUNK not 1 we would introduce mess:
+        // wrong chunk becomes 1, but it will be impossible to explicitly set
+        // to 1 because it becomes KMP_MIN_CHUNK...
+        // } else if ( chunk < KMP_MIN_CHUNK ) {
+        //   chunk = KMP_MIN_CHUNK;
       } else if (chunk > KMP_MAX_CHUNK) {
         chunk = KMP_MAX_CHUNK;
-      __kmp_msg(kmp_ms_warning, KMP_MSG(LargeChunk, name, comma),
+        __kmp_msg(kmp_ms_warning, KMP_MSG(LargeChunk, name, delim),
                   __kmp_msg_null);
         KMP_INFORM(Using_int_Value, name, chunk);
       }
-  } else if (ptr) {
-    SKIP_TOKEN(ptr);
     }
+  } else {
+    ptr = delim;
+  }
+
+  SCHEDULE_SET_MODIFIERS(sched, sched_modifier);
+
 #if KMP_USE_HIER_SCHED
   if (layer != kmp_hier_layer_e::LAYER_THREAD) {
     __kmp_hier_scheds.append(sched, chunk, layer);
   } else
 #endif
   {
     __kmp_chunk = chunk;
     __kmp_sched = sched;

 /* get the specified scheduling style */
 #if KMP_USE_HIER_SCHED
       if (!__kmp_strcasecmp_with_sentinel("EXPERIMENTAL", ptr, ' ')) {
         SKIP_TOKEN(ptr);
         SKIP_WS(ptr);
         while ((ptr = __kmp_parse_single_omp_schedule(name, ptr, true))) {
           while (*ptr == ' ' || *ptr == '\t' || *ptr == ':')
             ptr++;
+          if (*ptr == '\0')
+            break;
         }
       } else
 #endif
         __kmp_parse_single_omp_schedule(name, ptr);
     } else
       KMP_WARNING(EmptyString, name);
   }
 #if KMP_USE_HIER_SCHED
   __kmp_hier_scheds.sort();
 #endif
   K_DIAG(1, ("__kmp_static == %d\n", __kmp_static))
   K_DIAG(1, ("__kmp_guided == %d\n", __kmp_guided))
   K_DIAG(1, ("__kmp_sched == %d\n", __kmp_sched))
   K_DIAG(1, ("__kmp_chunk == %d\n", __kmp_chunk))
 } // __kmp_stg_parse_omp_schedule
 
 static void __kmp_stg_print_omp_schedule(kmp_str_buf_t *buffer,
                                          char const *name, void *data) {
   if (__kmp_env_format) {
     KMP_STR_BUF_PRINT_NAME_EX(name);
   } else {
     __kmp_str_buf_print(buffer, "   %s='", name);
   }
+  enum sched_type sched = SCHEDULE_WITHOUT_MODIFIERS(__kmp_sched);
+  if (SCHEDULE_HAS_MONOTONIC(__kmp_sched)) {
+    __kmp_str_buf_print(buffer, "monotonic:");
+  } else if (SCHEDULE_HAS_NONMONOTONIC(__kmp_sched)) {
+    __kmp_str_buf_print(buffer, "nonmonotonic:");
+  }
   if (__kmp_chunk) {
-    switch (__kmp_sched) {
+    switch (sched) {
     case kmp_sch_dynamic_chunked:
       __kmp_str_buf_print(buffer, "%s,%d'\n", "dynamic", __kmp_chunk);
       break;
     case kmp_sch_guided_iterative_chunked:
     case kmp_sch_guided_analytical_chunked:
       __kmp_str_buf_print(buffer, "%s,%d'\n", "guided", __kmp_chunk);
       break;
     case kmp_sch_trapezoidal:
       __kmp_str_buf_print(buffer, "%s,%d'\n", "trapezoidal", __kmp_chunk);
       break;
     case kmp_sch_static:
     case kmp_sch_static_chunked:
     case kmp_sch_static_balanced:
     case kmp_sch_static_greedy:
       __kmp_str_buf_print(buffer, "%s,%d'\n", "static", __kmp_chunk);
       break;
     case kmp_sch_static_steal:
       __kmp_str_buf_print(buffer, "%s,%d'\n", "static_steal", __kmp_chunk);
       break;
     case kmp_sch_auto:
       __kmp_str_buf_print(buffer, "%s,%d'\n", "auto", __kmp_chunk);
       break;
     }
   } else {
-    switch (__kmp_sched) {
+    switch (sched) {
     case kmp_sch_dynamic_chunked:
       __kmp_str_buf_print(buffer, "%s'\n", "dynamic");
       break;
     case kmp_sch_guided_iterative_chunked:
     case kmp_sch_guided_analytical_chunked:
       __kmp_str_buf_print(buffer, "%s'\n", "guided");
       break;
     case kmp_sch_trapezoidal:

openmp/trunk/runtime/test/worksharing/for/omp_monotonic_env.c

+// RUN: %libomp-compile
+// RUN: env OMP_SCHEDULE=monotonic:dynamic,50 %libomp-run monotonic dynamic 50
+// RUN: env OMP_SCHEDULE=monotonic:guided,51 %libomp-run monotonic guided 51
+// RUN: env OMP_SCHEDULE=monotonic:static,52 %libomp-run monotonic static 52
+// RUN: env OMP_SCHEDULE=nonmonotonic:dynamic,53 %libomp-run nonmonotonic dynamic 53
+// RUN: env OMP_SCHEDULE=nonmonotonic:guided,54 %libomp-run nonmonotonic guided 54
+
+// The test checks OMP 5.0 monotonic/nonmonotonic OMP_SCHEDULE parsing
+// The nonmonotonic tests see if the parser accepts nonmonotonic, if the
+// parser doesn't then a static schedule is assumed
+
+#include <stdio.h>
+#include <string.h>
+#include <omp.h>
+
+int err = 0;
+
+omp_sched_t sched_without_modifiers(omp_sched_t sched) {
+  return (omp_sched_t)((int)sched & ~((int)omp_sched_monotonic));
+}
+
+int sched_has_modifiers(omp_sched_t sched, omp_sched_t modifiers) {
+  return (int)sched & (int)modifiers;
+}
+
+// check that sched = hope | modifiers
+void check_schedule(const char *extra, const omp_sched_t sched, int chunk,
+                    omp_sched_t hope_sched, int hope_chunk) {
+
+  if (sched != hope_sched || chunk != hope_chunk) {
+    ++err;
+    printf("Error: %s: schedule: (%d, %d) is not equal to (%d, %d)\n", extra,
+           (int)hope_sched, hope_chunk, (int)sched, chunk);
+  }
+}
+
+omp_sched_t str2omp_sched(const char *str) {
+  if (!strcmp(str, "dynamic"))
+    return omp_sched_dynamic;
+  if (!strcmp(str, "static"))
+    return omp_sched_static;
+  if (!strcmp(str, "guided"))
+    return omp_sched_guided;
+  printf("Error: Unknown schedule type: %s\n", str);
+  exit(1);
+}
+
+int is_monotonic(const char *str) { return !strcmp(str, "monotonic"); }
+
+int main(int argc, char **argv) {
+  int i, monotonic, chunk, ref_chunk;
+  omp_sched_t sched, ref_sched;
+
+  if (argc != 4) {
+    printf("Error: usage: <executable> monotonic|nonmonotonic <schedule> "
+           "<chunk-size>\n");
+    exit(1);
+  }
+
+  monotonic = is_monotonic(argv[1]);
+  ref_sched = str2omp_sched(argv[2]);
+  ref_chunk = atoi(argv[3]);
+
+  omp_get_schedule(&sched, &chunk);
+
+  if (monotonic && !sched_has_modifiers(sched, omp_sched_monotonic)) {
+    printf("Error: sched (0x%x) does not have monotonic modifier\n",
+           (int)sched);
+    ++err;
+  }
+  sched = sched_without_modifiers(sched);
+  if (sched != ref_sched) {
+    printf("Error: sched (0x%x) is not 0x%x\n", (int)sched, (int)ref_sched);
+    ++err;
+  }
+  if (chunk != ref_chunk) {
+    printf("Error: chunk is not %d\n", ref_chunk);
+    ++err;
+  }
+  if (err > 0) {
+    printf("Failed\n");
+    return 1;
+  }
+  printf("Passed\n");
+  return 0;
+}

openmp/trunk/runtime/test/worksharing/for/omp_monotonic_schedule_set_get.c

+// RUN: %libomp-compile-and-run
+
+// The test checks OMP 5.0 monotonic/nonmonotonic scheduling API
+//   1. initial schedule should be (static,0)
+//   2. omp_get_schedule() should return the schedule set by omp_set_schedule()
+//   3. schedules set inside parallel should not impact outer tasks' schedules
+
+#include <stdio.h>
+#ifndef __INTEL_COMPILER
+#define _OMPIMP
+#endif
+
+#define NO_MODIFIERS ((omp_sched_t)0)
+
+#include "omp.h"
+
+int global = 0;
+int err = 0;
+
+omp_sched_t sched_append_modifiers(omp_sched_t sched, omp_sched_t modifiers) {
+  return (omp_sched_t)((int)sched | (int)modifiers);
+}
+
+omp_sched_t sched_without_modifiers(omp_sched_t sched) {
+  return (omp_sched_t)((int)sched & ~((int)omp_sched_monotonic));
+}
+
+int sched_has_modifiers(omp_sched_t sched, omp_sched_t modifiers) {
+  return (((int)sched & ((int)omp_sched_monotonic)) > 0);
+}
+
+// check that sched = hope | modifiers
+void check_schedule(const char *extra, const omp_sched_t sched, int chunk,
+                    omp_sched_t hope_sched, int hope_chunk) {
+
+  if (sched != hope_sched || chunk != hope_chunk) {
+#pragma omp atomic
+    ++err;
+    printf("Error: %s: schedule: (%d, %d) is not equal to (%d, %d)\n", extra,
+           (int)hope_sched, hope_chunk, (int)sched, chunk);
+  }
+}
+
+int main() {
+  int i;
+  int chunk;
+  omp_sched_t sched0;
+
+  omp_set_dynamic(0);
+  omp_set_nested(1);
+
+  // check serial region
+  omp_get_schedule(&sched0, &chunk);
+#ifdef DEBUG
+  printf("initial: (%d, %d)\n", sched0, chunk);
+#endif
+  check_schedule("initial", omp_sched_static, 0, sched0, chunk);
+  // set schedule before the parallel, check it after the parallel
+  omp_set_schedule(
+      sched_append_modifiers(omp_sched_dynamic, omp_sched_monotonic), 3);
+
+#pragma omp parallel num_threads(3) private(i)
+  {
+    omp_sched_t n_outer_set, n_outer_get;
+    int c_outer;
+    int tid = omp_get_thread_num();
+
+    n_outer_set = sched_append_modifiers((omp_sched_t)(tid + 1),
+                                         omp_sched_monotonic); // 1, 2, 3
+
+    // check outer parallel region
+    // master sets (static, unchunked), others - (dynamic, 1), (guided, 2)
+    // set schedule before inner parallel, check it after the parallel
+    omp_set_schedule(n_outer_set, tid);
+
+// Make sure this schedule doesn't crash the runtime
+#pragma omp for
+    for (i = 0; i < 100; ++i) {
+#pragma omp atomic
+      global++;
+    }
+
+#pragma omp parallel num_threads(3) private(i) shared(n_outer_set)
+    {
+      omp_sched_t n_inner_set, n_inner_get;
+      int c_inner_set, c_inner_get;
+      int tid = omp_get_thread_num();
+
+      n_inner_set = (omp_sched_t)(tid + 1); // 1, 2, 3
+      c_inner_set = (int)(n_outer_set)*10 +
+                    (int)n_inner_set; // 11, 12, 13, 21, 22, 23, 31, 32, 33
+      n_inner_set = sched_append_modifiers(n_inner_set, omp_sched_monotonic);
+      // schedules set inside parallel should not impact outer schedules
+      omp_set_schedule(n_inner_set, c_inner_set);
+
+// Make sure this schedule doesn't crash the runtime
+#pragma omp for
+      for (i = 0; i < 100; ++i) {
+#pragma omp atomic
+        global++;
+      }
+
+#pragma omp barrier
+      omp_get_schedule(&n_inner_get, &c_inner_get);
+#ifdef DEBUG
+      printf("inner parallel: o_th %d, i_th %d, (%d, %d)\n", n_outer_set - 1,
+             tid, n_inner_get, c_inner_get);
+#endif
+      check_schedule("inner", n_inner_set, c_inner_set, n_inner_get,
+                     c_inner_get);
+    }
+
+    omp_get_schedule(&n_outer_get, &c_outer);
+#ifdef DEBUG
+    printf("outer parallel: thread %d, (%d, %d)\n", tid, n_outer_get, c_outer);
+#endif
+    check_schedule("outer", n_outer_set, tid, n_outer_get, c_outer);
+  }
+
+  omp_get_schedule(&sched0, &chunk);
+#ifdef DEBUG
+  printf("after parallels: (%d, %d)\n", sched0, chunk);
+#endif
+  check_schedule("after parallels",
+                 sched_append_modifiers(omp_sched_dynamic, omp_sched_monotonic),
+                 3, sched0, chunk);
+
+  if (err > 0) {
+    printf("Failed\n");
+    return 1;
+  }
+  printf("Passed\n");
+  return 0;
+}