Code cleanup for the runtime without monitor thread

[hbae]

Summary

This change removes/disables unnecessary code when monitor thread is not used.

[AndreyChurbanov]

LGTM

openmp/trunk/runtime/src/kmp.h

 # define KMP_DEFAULT_STKSIZE     ((size_t)(2 * 1024 * 1024))
 #elif KMP_ARCH_X86_64
 # define KMP_DEFAULT_STKSIZE     ((size_t)(4 * 1024 * 1024))
 # define KMP_BACKUP_STKSIZE      ((size_t)(2 * 1024 * 1024))
 #else
 # define KMP_DEFAULT_STKSIZE     ((size_t)(1024 * 1024))
 #endif
 
-#define KMP_DEFAULT_MONITOR_STKSIZE     ((size_t)(64 * 1024))
-
 #define KMP_DEFAULT_MALLOC_POOL_INCR    ((size_t) (1024 * 1024))
 #define KMP_MIN_MALLOC_POOL_INCR        ((size_t) (4 * 1024))
 #define KMP_MAX_MALLOC_POOL_INCR        (~((size_t)1<<((sizeof(size_t)*(1<<3))-1)))
 
 #define KMP_MIN_STKOFFSET       (0)
 #define KMP_MAX_STKOFFSET       KMP_MAX_STKSIZE
 #if KMP_OS_DARWIN
 # define KMP_DEFAULT_STKOFFSET  KMP_MIN_STKOFFSET
 #else
 # define KMP_DEFAULT_STKOFFSET  CACHE_LINE
 #endif
 
 #define KMP_MIN_STKPADDING      (0)
 #define KMP_MAX_STKPADDING      (2 * 1024 * 1024)
 
-#define KMP_MIN_MONITOR_WAKEUPS      (1)       /* min number of times monitor wakes up per second */
-#define KMP_MAX_MONITOR_WAKEUPS      (1000)    /* maximum number of times monitor can wake up per second */
 #define KMP_BLOCKTIME_MULTIPLIER     (1000)    /* number of blocktime units per second */
 #define KMP_MIN_BLOCKTIME            (0)
 #define KMP_MAX_BLOCKTIME            (INT_MAX) /* Must be this for "infinite" setting the work */
 #define KMP_DEFAULT_BLOCKTIME        (200)     /*  __kmp_blocktime is in milliseconds  */
+
+#if KMP_USE_MONITOR
+#define KMP_DEFAULT_MONITOR_STKSIZE  ((size_t)(64 * 1024))
+#define KMP_MIN_MONITOR_WAKEUPS      (1)       /* min number of times monitor wakes up per second */
+#define KMP_MAX_MONITOR_WAKEUPS      (1000)    /* maximum number of times monitor can wake up per second */
+
 /* Calculate new number of monitor wakeups for a specific block time based on previous monitor_wakeups */
 /* Only allow increasing number of wakeups */
 #define KMP_WAKEUPS_FROM_BLOCKTIME(blocktime, monitor_wakeups) \
                                  ( ((blocktime) == KMP_MAX_BLOCKTIME) ? (monitor_wakeups) : \
                                    ((blocktime) == KMP_MIN_BLOCKTIME) ? KMP_MAX_MONITOR_WAKEUPS : \
                                    ((monitor_wakeups) > (KMP_BLOCKTIME_MULTIPLIER / (blocktime))) ? (monitor_wakeups) : \
                                        (KMP_BLOCKTIME_MULTIPLIER) / (blocktime) )
 
 /* Calculate number of intervals for a specific block time based on monitor_wakeups */
 #define KMP_INTERVALS_FROM_BLOCKTIME(blocktime, monitor_wakeups)  \
                                  ( ( (blocktime) + (KMP_BLOCKTIME_MULTIPLIER / (monitor_wakeups)) - 1 ) /  \
                                    (KMP_BLOCKTIME_MULTIPLIER / (monitor_wakeups)) )
+#endif // KMP_USE_MONITOR
 
 #define KMP_MIN_STATSCOLS       40
 #define KMP_MAX_STATSCOLS       4096
 #define KMP_DEFAULT_STATSCOLS   80
 
 #define KMP_MIN_INTERVAL        0
 #define KMP_MAX_INTERVAL        (INT_MAX-1)
 #define KMP_DEFAULT_INTERVAL    0

 
 /* Record for holding the values of the internal controls stack records */
 typedef struct kmp_internal_control {
     int           serial_nesting_level;  /* corresponds to the value of the th_team_serialized field */
     kmp_int8      nested;                /* internal control for nested parallelism (per thread) */
     kmp_int8      dynamic;               /* internal control for dynamic adjustment of threads (per thread) */
     kmp_int8      bt_set;                /* internal control for whether blocktime is explicitly set */
     int           blocktime;             /* internal control for blocktime */
+#if KMP_USE_MONITOR
     int           bt_intervals;          /* internal control for blocktime intervals */
+#endif
     int           nproc;                 /* internal control for #threads for next parallel region (per thread) */
     int           max_active_levels;     /* internal control for max_active_levels */
     kmp_r_sched_t sched;                 /* internal control for runtime schedule {sched,chunk} pair */
 #if OMP_40_ENABLED
     kmp_proc_bind_t proc_bind;           /* internal control for affinity  */
     kmp_int32       default_device;      /* internal control for default device */
 #endif // OMP_40_ENABLED
     struct kmp_internal_control *next;

 
 } kmp_local_t;
 
 #define KMP_CHECK_UPDATE(a, b) if ((a) != (b)) (a) = (b)
 #define KMP_CHECK_UPDATE_SYNC(a, b) if ((a) != (b)) TCW_SYNC_PTR((a), (b))
 
 #define get__blocktime( xteam, xtid )     ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.blocktime)
 #define get__bt_set( xteam, xtid )        ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.bt_set)
+#if KMP_USE_MONITOR
 #define get__bt_intervals( xteam, xtid )  ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.bt_intervals)
+#endif
 
 #define get__nested_2(xteam,xtid)         ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.nested)
 #define get__dynamic_2(xteam,xtid)        ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.dynamic)
 #define get__nproc_2(xteam,xtid)          ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.nproc)
 #define get__sched_2(xteam,xtid)          ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.sched)
 
 #define set__blocktime_team( xteam, xtid, xval ) \
         ( ( (xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.blocktime )    = (xval) )
 
+#if KMP_USE_MONITOR
 #define set__bt_intervals_team( xteam, xtid, xval ) \
         ( ( (xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.bt_intervals ) = (xval) )
+#endif
 
 #define set__bt_set_team( xteam, xtid, xval ) \
         ( ( (xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.bt_set )       = (xval) )
 
 
 #define set__nested( xthread, xval )                            \
         ( ( (xthread)->th.th_current_task->td_icvs.nested ) = (xval) )
 #define get__nested( xthread ) \
#if OMP_40_ENABLED
     int               th_teams_level;     /* save initial level of teams construct */
                                           /* it is 0 on device but may be any on host */
 #endif
 
     /* The blocktime info is copied from the team struct to the thread sruct */
     /* at the start of a barrier, and the values stored in the team are used */
     /* at points in the code where the team struct is no longer guaranteed   */
     /* to exist (from the POV of worker threads).                            */
+#if KMP_USE_MONITOR
     int               th_team_bt_intervals;
+#endif
     int               th_team_bt_set;
 
 
 #if KMP_AFFINITY_SUPPORTED
     kmp_affin_mask_t  *th_affin_mask; /* thread's current affinity mask */
 #endif
 
 /*

 extern int        __kmp_max_nth;        /* maximum total number of concurrently-existing threads */
 extern int        __kmp_threads_capacity; /* capacity of the arrays __kmp_threads and __kmp_root */
 extern int        __kmp_dflt_team_nth;  /* default number of threads in a parallel region a la OMP_NUM_THREADS */
 extern int        __kmp_dflt_team_nth_ub; /* upper bound on "" determined at serial initialization */
 extern int        __kmp_tp_capacity;    /* capacity of __kmp_threads if threadprivate is used (fixed) */
 extern int        __kmp_tp_cached;      /* whether threadprivate cache has been created (__kmpc_threadprivate_cached()) */
 extern int        __kmp_dflt_nested;    /* nested parallelism enabled by default a la OMP_NESTED */
 extern int        __kmp_dflt_blocktime; /* number of milliseconds to wait before blocking (env setting) */
+#if KMP_USE_MONITOR
 extern int        __kmp_monitor_wakeups;/* number of times monitor wakes up per second */
 extern int        __kmp_bt_intervals;   /* number of monitor timestamp intervals before blocking */
+#endif
 #ifdef KMP_ADJUST_BLOCKTIME
 extern int        __kmp_zero_bt;        /* whether blocktime has been forced to zero */
 #endif /* KMP_ADJUST_BLOCKTIME */
 #ifdef KMP_DFLT_NTH_CORES
 extern int        __kmp_ncores;         /* Total number of cores for threads placement */
 #endif
 extern int        __kmp_abort_delay;    /* Number of millisecs to delay on abort for VTune */
 

openmp/trunk/runtime/src/kmp_barrier.cpp

#endif /* USE_ITT_BUILD */
             KA_TRACE(15, ("__kmp_barrier: T#%d(%d:%d) past tasking barrier\n",
                           gtid, __kmp_team_from_gtid(gtid)->t.t_id, __kmp_tid_from_gtid(gtid)));
         }
 
         /* Copy the blocktime info to the thread, where __kmp_wait_template() can access it when
            the team struct is not guaranteed to exist. */
         // See note about the corresponding code in __kmp_join_barrier() being performance-critical.
         if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
+#if KMP_USE_MONITOR
             this_thr->th.th_team_bt_intervals = team->t.t_implicit_task_taskdata[tid].td_icvs.bt_intervals;
+#endif
             this_thr->th.th_team_bt_set = team->t.t_implicit_task_taskdata[tid].td_icvs.bt_set;
         }
 
 #if USE_ITT_BUILD
         if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
             __kmp_itt_barrier_starting(gtid, itt_sync_obj);
 #endif /* USE_ITT_BUILD */
 #if USE_DEBUGGER
# ifdef KMP_DEBUG
     }
 # endif /* KMP_DEBUG */
 
     /* Copy the blocktime info to the thread, where __kmp_wait_template() can access it when the
        team struct is not guaranteed to exist. Doing these loads causes a cache miss slows
        down EPCC parallel by 2x. As a workaround, we do not perform the copy if blocktime=infinite,
        since the values are not used by __kmp_wait_template() in that case. */
     if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
+#if KMP_USE_MONITOR
         this_thr->th.th_team_bt_intervals = team->t.t_implicit_task_taskdata[tid].td_icvs.bt_intervals;
+#endif
         this_thr->th.th_team_bt_set = team->t.t_implicit_task_taskdata[tid].td_icvs.bt_set;
     }
 
 #if USE_ITT_BUILD
     if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
         __kmp_itt_barrier_starting(gtid, itt_sync_obj);
 #endif /* USE_ITT_BUILD */
 
#endif
             __kmp_task_team_setup(this_thr, team, 0);  // 0 indicates setup current task team if nthreads > 1
         }
 
         /* The master thread may have changed its blocktime between the join barrier and the
            fork barrier. Copy the blocktime info to the thread, where __kmp_wait_template() can
            access it when the team struct is not guaranteed to exist. */
         // See note about the corresponding code in __kmp_join_barrier() being performance-critical
         if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
+#if KMP_USE_MONITOR
             this_thr->th.th_team_bt_intervals = team->t.t_implicit_task_taskdata[tid].td_icvs.bt_intervals;
+#endif
             this_thr->th.th_team_bt_set = team->t.t_implicit_task_taskdata[tid].td_icvs.bt_set;
         }
     } // master
 
     switch (__kmp_barrier_release_pattern[bs_forkjoin_barrier]) {
     case bp_hyper_bar: {
         KMP_ASSERT(__kmp_barrier_release_branch_bits[bs_forkjoin_barrier]);
         __kmp_hyper_barrier_release(bs_forkjoin_barrier, this_thr, gtid, tid, TRUE

openmp/trunk/runtime/src/kmp_global.c

 int __kmp_hot_teams_max_level    = 1; /* nesting level of hot teams */
 #endif
 enum library_type __kmp_library = library_none;
 enum sched_type     __kmp_sched = kmp_sch_default;  /* scheduling method for runtime scheduling */
 enum sched_type    __kmp_static = kmp_sch_static_greedy; /* default static scheduling method */
 enum sched_type    __kmp_guided = kmp_sch_guided_iterative_chunked; /* default guided scheduling method */
 enum sched_type      __kmp_auto = kmp_sch_guided_analytical_chunked; /* default auto scheduling method */
 int        __kmp_dflt_blocktime = KMP_DEFAULT_BLOCKTIME;
+#if KMP_USE_MONITOR
 int       __kmp_monitor_wakeups = KMP_MIN_MONITOR_WAKEUPS;
 int          __kmp_bt_intervals = KMP_INTERVALS_FROM_BLOCKTIME( KMP_DEFAULT_BLOCKTIME, KMP_MIN_MONITOR_WAKEUPS );
+#endif
 #ifdef KMP_ADJUST_BLOCKTIME
 int               __kmp_zero_bt = FALSE;
 #endif /* KMP_ADJUST_BLOCKTIME */
 #ifdef KMP_DFLT_NTH_CORES
 int                __kmp_ncores = 0;
 #endif
 int                 __kmp_chunk = 0;
 int           __kmp_abort_delay = 0;

openmp/trunk/runtime/src/kmp_runtime.c

 #endif /* OMP_40_ENABLED */
 
     kmp_internal_control_t g_icvs = {
       0,                            //int serial_nesting_level; //corresponds to the value of the th_team_serialized field
       (kmp_int8)__kmp_dflt_nested,            //int nested;               //internal control for nested parallelism (per thread)
       (kmp_int8)__kmp_global.g.g_dynamic,                                 //internal control for dynamic adjustment of threads (per thread)
       (kmp_int8)__kmp_env_blocktime,          //int bt_set;               //internal control for whether blocktime is explicitly set
       __kmp_dflt_blocktime,         //int blocktime;            //internal control for blocktime
+#if KMP_USE_MONITOR
       __kmp_bt_intervals,           //int bt_intervals;         //internal control for blocktime intervals
+#endif
       __kmp_dflt_team_nth,          //int nproc;                //internal control for # of threads for next parallel region (per thread)
                                     // (use a max ub on value if __kmp_parallel_initialize not called yet)
       __kmp_dflt_max_active_levels, //int max_active_levels;    //internal control for max_active_levels
       r_sched,                      //kmp_r_sched_t sched;      //internal control for runtime schedule {sched,chunk} pair
 #if OMP_40_ENABLED
       __kmp_nested_proc_bind.bind_types[0],
       __kmp_default_device,
 #endif /* OMP_40_ENABLED */
#endif
     }
     if( __kmp_dflt_team_nth_ub > __kmp_sys_max_nth ) {
         __kmp_dflt_team_nth_ub = __kmp_sys_max_nth;
     }
     __kmp_max_nth = __kmp_sys_max_nth;
 
     // Three vars below moved here from __kmp_env_initialize() "KMP_BLOCKTIME" part
     __kmp_dflt_blocktime = KMP_DEFAULT_BLOCKTIME;
+#if KMP_USE_MONITOR
     __kmp_monitor_wakeups = KMP_WAKEUPS_FROM_BLOCKTIME( __kmp_dflt_blocktime, __kmp_monitor_wakeups );
     __kmp_bt_intervals = KMP_INTERVALS_FROM_BLOCKTIME( __kmp_dflt_blocktime, __kmp_monitor_wakeups );
+#endif
     // From "KMP_LIBRARY" part of __kmp_env_initialize()
     __kmp_library = library_throughput;
     // From KMP_SCHEDULE initialization
     __kmp_static = kmp_sch_static_balanced;
     // AC: do not use analytical here, because it is non-monotonous
     //__kmp_guided = kmp_sch_guided_iterative_chunked;
     //__kmp_auto = kmp_sch_guided_analytical_chunked; // AC: it is the default, no need to repeate assignment
     // Barrier initialization. Moved here from __kmp_env_initialize() Barrier branch bit control and barrier method

 
 /* ------------------------------------------------------------------------ */
 /* ------------------------------------------------------------------------ */
 
 void
 __kmp_aux_set_blocktime (int arg, kmp_info_t *thread, int tid)
 {
     int blocktime = arg;        /* argument is in milliseconds */
+#if KMP_USE_MONITOR
     int bt_intervals;
+#endif
     int bt_set;
 
     __kmp_save_internal_controls( thread );
 
     /* Normalize and set blocktime for the teams */
     if (blocktime < KMP_MIN_BLOCKTIME)
         blocktime = KMP_MIN_BLOCKTIME;
     else if (blocktime > KMP_MAX_BLOCKTIME)
         blocktime = KMP_MAX_BLOCKTIME;
 
     set__blocktime_team( thread->th.th_team, tid, blocktime );
     set__blocktime_team( thread->th.th_serial_team, 0, blocktime );
 
+#if KMP_USE_MONITOR
     /* Calculate and set blocktime intervals for the teams */
     bt_intervals = KMP_INTERVALS_FROM_BLOCKTIME(blocktime, __kmp_monitor_wakeups);
 
     set__bt_intervals_team( thread->th.th_team, tid, bt_intervals );
     set__bt_intervals_team( thread->th.th_serial_team, 0, bt_intervals );
+#endif
 
     /* Set whether blocktime has been set to "TRUE" */
     bt_set = TRUE;
 
     set__bt_set_team( thread->th.th_team, tid, bt_set );
     set__bt_set_team( thread->th.th_serial_team, 0, bt_set );
-    KF_TRACE(10, ( "kmp_set_blocktime: T#%d(%d:%d), blocktime=%d, bt_intervals=%d, monitor_updates=%d\n",
-                  __kmp_gtid_from_tid(tid, thread->th.th_team),
-                  thread->th.th_team->t.t_id, tid, blocktime, bt_intervals, __kmp_monitor_wakeups ) );
+    KF_TRACE(10, ( "kmp_set_blocktime: T#%d(%d:%d), blocktime=%d"
+#if KMP_USE_MONITOR
+                   ", bt_intervals=%d, monitor_updates=%d"
+#endif
+                   "\n",
+                   __kmp_gtid_from_tid(tid, thread->th.th_team), thread->th.th_team->t.t_id, tid, blocktime
+#if KMP_USE_MONITOR
+                   , bt_intervals, __kmp_monitor_wakeups
+#endif
+                 ) );
 }
 
 void
 __kmp_aux_set_defaults(
     char const * str,
     int          len
 ) {
     if ( ! __kmp_init_serial ) {

openmp/trunk/runtime/src/kmp_settings.c

if ( __kmp_dflt_blocktime < 0 ) {
             KMP_INFORM( MinValueUsing, name, __kmp_dflt_blocktime );
         } else if ( __kmp_dflt_blocktime > KMP_MAX_BLOCKTIME ) {
             __kmp_dflt_blocktime = KMP_MAX_BLOCKTIME;
             __kmp_msg( kmp_ms_warning, KMP_MSG( LargeValue, name, value ), __kmp_msg_null );
             KMP_INFORM( MaxValueUsing, name, __kmp_dflt_blocktime );
         }; // if
         __kmp_env_blocktime = TRUE;    // KMP_BLOCKTIME was specified.
     }; // if
-    // calculate number of monitor thread wakeup intervals corresonding to blocktime.
+#if KMP_USE_MONITOR
+    // calculate number of monitor thread wakeup intervals corresponding to blocktime.
     __kmp_monitor_wakeups = KMP_WAKEUPS_FROM_BLOCKTIME( __kmp_dflt_blocktime, __kmp_monitor_wakeups );
     __kmp_bt_intervals = KMP_INTERVALS_FROM_BLOCKTIME( __kmp_dflt_blocktime, __kmp_monitor_wakeups );
+#endif
     K_DIAG( 1, ( "__kmp_env_blocktime == %d\n", __kmp_env_blocktime ) );
     if ( __kmp_env_blocktime ) {
         K_DIAG( 1, ( "__kmp_dflt_blocktime == %d\n", __kmp_dflt_blocktime ) );
     }
 } // __kmp_stg_parse_blocktime
 
 static void
 __kmp_stg_print_blocktime( kmp_str_buf_t * buffer, char const * name, void * data ) {