Fix performance issue incurred by removing monitor thread.

[hbae]

Summary

This change fixes a performance issue incurred by removing monitor thread.

When the monitor thread is used, most threads in the team directly go to
sleep if the copy of bt_intervals/bt_set is not available in the cache,
and this happens at least once per thread in the wait function, making the
overall performance slightly better.
This change tries to mimic this behavior by using the bt_intervals cache,
which simply keeps the blocktime interval in terms of the platform-dependent
ticks or nanoseconds.

[jlpeyton]

LGTM.

openmp/trunk/runtime/src/kmp.h

#define KMP_WAKEUPS_FROM_BLOCKTIME(blocktime, monitor_wakeups) \
                                    ((blocktime) == KMP_MIN_BLOCKTIME) ? KMP_MAX_MONITOR_WAKEUPS : \
                                    ((monitor_wakeups) > (KMP_BLOCKTIME_MULTIPLIER / (blocktime))) ? (monitor_wakeups) : \
                                        (KMP_BLOCKTIME_MULTIPLIER) / (blocktime) )
 
 /* Calculate number of intervals for a specific block time based on monitor_wakeups */
 #define KMP_INTERVALS_FROM_BLOCKTIME(blocktime, monitor_wakeups)  \
                                  ( ( (blocktime) + (KMP_BLOCKTIME_MULTIPLIER / (monitor_wakeups)) - 1 ) /  \
                                    (KMP_BLOCKTIME_MULTIPLIER / (monitor_wakeups)) )
+#else
+# if KMP_OS_UNIX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
+   // HW TSC is used to reduce overhead (clock tick instead of nanosecond).
+   extern double __kmp_ticks_per_nsec;
+#  define KMP_NOW() __kmp_hardware_timestamp()
+#  define KMP_BLOCKTIME_INTERVAL() (__kmp_dflt_blocktime * KMP_USEC_PER_SEC * __kmp_ticks_per_nsec)
+#  define KMP_BLOCKING(goal, count) ((goal) > KMP_NOW())
+# else
+   // System time is retrieved sporadically while blocking.
+   extern kmp_uint64 __kmp_now_nsec();
+#  define KMP_NOW() __kmp_now_nsec()
+#  define KMP_BLOCKTIME_INTERVAL() (__kmp_dflt_blocktime * KMP_USEC_PER_SEC)
+#  define KMP_BLOCKING(goal, count) ((count) % 1000 != 0 || (goal) > KMP_NOW())
+# endif
 #endif // KMP_USE_MONITOR
 
 #define KMP_MIN_STATSCOLS       40
 #define KMP_MAX_STATSCOLS       4096
 #define KMP_DEFAULT_STATSCOLS   80
 
 #define KMP_MIN_INTERVAL        0
 #define KMP_MAX_INTERVAL        (INT_MAX-1)

 #endif
 
     /* The blocktime info is copied from the team struct to the thread sruct */
     /* at the start of a barrier, and the values stored in the team are used */
     /* at points in the code where the team struct is no longer guaranteed   */
     /* to exist (from the POV of worker threads).                            */
 #if KMP_USE_MONITOR
     int               th_team_bt_intervals;
-#endif
     int               th_team_bt_set;
+#else
+    kmp_uint64        th_team_bt_intervals;
+#endif
 
 
 #if KMP_AFFINITY_SUPPORTED
     kmp_affin_mask_t  *th_affin_mask; /* thread's current affinity mask */
 #endif
 
 /*
  * The data set by the master at reinit, then R/W by the worker

openmp/trunk/runtime/src/kmp_barrier.cpp

#endif /* USE_ITT_BUILD */
         }
 
         /* Copy the blocktime info to the thread, where __kmp_wait_template() can access it when
            the team struct is not guaranteed to exist. */
         // See note about the corresponding code in __kmp_join_barrier() being performance-critical.
         if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
 #if KMP_USE_MONITOR
             this_thr->th.th_team_bt_intervals = team->t.t_implicit_task_taskdata[tid].td_icvs.bt_intervals;
-#endif
             this_thr->th.th_team_bt_set = team->t.t_implicit_task_taskdata[tid].td_icvs.bt_set;
+#else
+            this_thr->th.th_team_bt_intervals = KMP_BLOCKTIME_INTERVAL();
+#endif
         }
 
 #if USE_ITT_BUILD
         if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
             __kmp_itt_barrier_starting(gtid, itt_sync_obj);
 #endif /* USE_ITT_BUILD */
 #if USE_DEBUGGER
         // Let the debugger know: the thread arrived to the barrier and waiting.
# endif /* KMP_DEBUG */
 
     /* Copy the blocktime info to the thread, where __kmp_wait_template() can access it when the
        team struct is not guaranteed to exist. Doing these loads causes a cache miss slows
        down EPCC parallel by 2x. As a workaround, we do not perform the copy if blocktime=infinite,
        since the values are not used by __kmp_wait_template() in that case. */
     if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
 #if KMP_USE_MONITOR
         this_thr->th.th_team_bt_intervals = team->t.t_implicit_task_taskdata[tid].td_icvs.bt_intervals;
-#endif
         this_thr->th.th_team_bt_set = team->t.t_implicit_task_taskdata[tid].td_icvs.bt_set;
+#else
+        this_thr->th.th_team_bt_intervals = KMP_BLOCKTIME_INTERVAL();
+#endif
     }
 
 #if USE_ITT_BUILD
     if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
         __kmp_itt_barrier_starting(gtid, itt_sync_obj);
 #endif /* USE_ITT_BUILD */
 
     switch (__kmp_barrier_gather_pattern[bs_forkjoin_barrier]) {
#endif
 
         /* The master thread may have changed its blocktime between the join barrier and the
            fork barrier. Copy the blocktime info to the thread, where __kmp_wait_template() can
            access it when the team struct is not guaranteed to exist. */
         // See note about the corresponding code in __kmp_join_barrier() being performance-critical
         if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
 #if KMP_USE_MONITOR
             this_thr->th.th_team_bt_intervals = team->t.t_implicit_task_taskdata[tid].td_icvs.bt_intervals;
-#endif
             this_thr->th.th_team_bt_set = team->t.t_implicit_task_taskdata[tid].td_icvs.bt_set;
+#else
+            this_thr->th.th_team_bt_intervals = KMP_BLOCKTIME_INTERVAL();
+#endif
         }
     } // master
 
     switch (__kmp_barrier_release_pattern[bs_forkjoin_barrier]) {
     case bp_hyper_bar: {
         KMP_ASSERT(__kmp_barrier_release_branch_bits[bs_forkjoin_barrier]);
         __kmp_hyper_barrier_release(bs_forkjoin_barrier, this_thr, gtid, tid, TRUE
                                     USE_ITT_BUILD_ARG(itt_sync_obj) );

openmp/trunk/runtime/src/kmp_wait_release.h

public:
     bool is_sleeping();
     bool is_any_sleeping();
     bool is_sleeping_val(P old_loc);
     int execute_tasks(kmp_info_t *this_thr, kmp_int32 gtid, int final_spin, int *thread_finished
                       USE_ITT_BUILD_ARG(void * itt_sync_obj), kmp_int32 is_constrained);
     */
 };
 
-#if ! KMP_USE_MONITOR
-# if KMP_OS_UNIX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
-   // HW TSC is used to reduce overhead (clock tick instead of nanosecond).
-   extern double __kmp_ticks_per_nsec;
-#  define KMP_NOW() __kmp_hardware_timestamp()
-#  define KMP_BLOCKTIME_INTERVAL() (__kmp_dflt_blocktime * KMP_USEC_PER_SEC * __kmp_ticks_per_nsec)
-#  define KMP_BLOCKING(goal, count) ((goal) > KMP_NOW())
-# else
-   // System time is retrieved sporadically while blocking.
-   extern kmp_uint64 __kmp_now_nsec();
-#  define KMP_NOW() __kmp_now_nsec()
-#  define KMP_BLOCKTIME_INTERVAL() (__kmp_dflt_blocktime * KMP_USEC_PER_SEC)
-#  define KMP_BLOCKING(goal, count) ((count) % 1000 != 0 || (goal) > KMP_NOW())
-# endif
-#endif
-
 /* Spin wait loop that first does pause, then yield, then sleep. A thread that calls __kmp_wait_*
    must make certain that another thread calls __kmp_release to wake it back up to prevent deadlocks!  */
 template <class C>
 static inline void
 __kmp_wait_template(kmp_info_t *this_thr, C *flag, int final_spin
                     USE_ITT_BUILD_ARG(void * itt_sync_obj) )
 {
     // NOTE: We may not belong to a team at this point.
#endif /* KMP_ADJUST_BLOCKTIME */
             hibernate++;
 
         // Add in the current time value.
         hibernate += TCR_4(__kmp_global.g.g_time.dt.t_value);
         KF_TRACE(20, ("__kmp_wait_sleep: T#%d now=%d, hibernate=%d, intervals=%d\n",
                       th_gtid, __kmp_global.g.g_time.dt.t_value, hibernate,
                       hibernate - __kmp_global.g.g_time.dt.t_value));
 #else
-        hibernate_goal = KMP_NOW() + KMP_BLOCKTIME_INTERVAL();
+        hibernate_goal = KMP_NOW() + this_thr->th.th_team_bt_intervals;
         poll_count = 0;
 #endif // KMP_USE_MONITOR
     }
 
     oversubscribed = (TCR_4(__kmp_nth) > __kmp_avail_proc);
     KMP_MB();
 
     // Main wait spin loop