Add new OpenMP 4.5 schedule clause modifiers (monotonic/non-monotonic) feature

[jlpeyton]

Summary

The monotonic/non-monotonic flags are sent to the runtime via the sched_type by
setting the 30th (non-monotonic) or 29th (monotonic) bit in the sched_type.
Macros are added to probe if monotonic or non-monotonic is specified
(SCHEDULE_HAS_[NON]MONOTONIC & SCHEDULE_HAS_NO_MODIFIERS)
and also to to get the base sched_type (SCHEDULE_WITHOUT_MODIFIERS)

Currently, nothing is done with the modifiers.

Also, this patch adds some comments on the use of the enumerations in at least one place where it is subtle.

[AndreyChurbanov]

LGTM

runtime/src/kmp.h

#endif
     kmp_nm_ord_static                 = 194,  /**< ordered static unspecialized */
     kmp_nm_ord_dynamic_chunked        = 195,
     kmp_nm_ord_guided_chunked         = 196,
     kmp_nm_ord_runtime                = 197,
     kmp_nm_ord_auto                   = 198,  /**< auto */
     kmp_nm_ord_trapezoidal            = 199,
     kmp_nm_upper                      = 200,  /**< upper bound for nomerge values */
 
+#if OMP_41_ENABLED
+    /* Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers.
+     * Since we need to distinguish the three possible cases (no modifier, monotonic modifier,
+     * nonmonotonic modifier), we need separate bits for each modifier.
+     * The absence of monotonic does not imply nonmonotonic, especially since 4.5 says
+     * that the behaviour of the "no modifier" case is implementation defined in 4.5,
+     * but will become "nonmonotonic" in 5.0.
+     *
+     * Since we're passing a full 32 bit value, we can use a couple of high bits for these
+     * flags; out of paranoia we avoid the sign bit.
+     *
+     * These modifiers can be or-ed into non-static schedules by the compiler to pass
+     * the additional information.
+     * They will be stripped early in the processing in __kmp_dispatch_init when setting up schedules, so
+     * most of the code won't ever see schedules with these bits set.
+     */
+    kmp_sch_modifier_monotonic      = (1<<29), /**< Set if the monotonic schedule modifier was present */
+    kmp_sch_modifier_nonmonotonic   = (1<<30), /**< Set if the nonmonotonic schedule modifier was present */
+
+# define SCHEDULE_WITHOUT_MODIFIERS(s) (enum sched_type)((s) & ~ (kmp_sch_modifier_nonmonotonic | kmp_sch_modifier_monotonic))
+# define SCHEDULE_HAS_MONOTONIC(s)     (((s) & kmp_sch_modifier_monotonic)    != 0)
+# define SCHEDULE_HAS_NONMONOTONIC(s)  (((s) & kmp_sch_modifier_nonmonotonic) != 0)
+# define SCHEDULE_HAS_NO_MODIFIERS(s)  (((s) & (kmp_sch_modifier_nonmonotonic | kmp_sch_modifier_monotonic)) == 0)
+#else
+    /* By doing this we hope to avoid multiple tests on OMP_41_ENABLED. Compilers can now eliminate tests on compile time
+     * constants and dead code that results from them, so we can leave code guarded by such an if in place.
+     */
+# define SCHEDULE_WITHOUT_MODIFIERS(s) (s)
+# define SCHEDULE_HAS_MONOTONIC(s)     false
+# define SCHEDULE_HAS_NONMONOTONIC(s)  false
+# define SCHEDULE_HAS_NO_MODIFIERS(s)  true
+#endif
+
     kmp_sch_default = kmp_sch_static  /**< default scheduling algorithm */
 };
 
 /* Type to keep runtime schedule set via OMP_SCHEDULE or omp_set_schedule() */
 typedef struct kmp_r_sched {
     enum sched_type r_sched_type;
     int             chunk;
 } kmp_r_sched_t;

runtime/src/kmp_dispatch.cpp

if ( ! active ) {
 
         /* What happens when number of threads changes, need to resize buffer? */
         pr = reinterpret_cast< dispatch_private_info_template< T >  * >
             ( &th -> th.th_dispatch -> th_disp_buffer[ my_buffer_index % KMP_MAX_DISP_BUF ] );
         sh = reinterpret_cast< dispatch_shared_info_template< UT > volatile * >
             ( &team -> t.t_disp_buffer[ my_buffer_index % KMP_MAX_DISP_BUF ] );
     }
 
+    /* Currently just ignore the monotonic and non-monotonic modifiers (the compiler isn't producing them
+     * yet anyway).
+     * When it is we'll want to look at them somewhere here and use that information to add to our
+     * schedule choice. We shouldn't need to pass them on, they merely affect which schedule we can
+     * legally choose for various dynamic cases. (In paritcular, whether or not a stealing scheme is legal).
+     */
+    schedule = SCHEDULE_WITHOUT_MODIFIERS(schedule);
+
     /* Pick up the nomerge/ordered bits from the scheduling type */
     if ( (schedule >= kmp_nm_lower) && (schedule < kmp_nm_upper) ) {
         pr->nomerge = TRUE;
         schedule = (enum sched_type)(((int)schedule) - (kmp_nm_lower - kmp_sch_lower));
     } else {
         pr->nomerge = FALSE;
     }
     pr->type_size = ___kmp_size_type; // remember the size of variables

runtime/src/kmp_sched.cpp

#if OMPT_SUPPORT && OMPT_TRACE
                 team_info->microtask);
         }
 #endif
         KMP_COUNT_VALUE (FOR_static_iterations, 0);
         return;
     }
 
     #if OMP_40_ENABLED
+    // Although there are schedule enumerations above kmp_ord_upper which are not schedules for "distribute",
+    // the only ones which are useful are dynamic, so cannot be seen here, since this codepath is only executed
+    // for static schedules.
     if ( schedtype > kmp_ord_upper ) {
         // we are in DISTRIBUTE construct
         schedtype += kmp_sch_static - kmp_distribute_static;      // AC: convert to usual schedule type
         tid  = th->th.th_team->t.t_master_tid;
         team = th->th.th_team->t.t_parent;
     } else
     #endif
     {