Improving EPCC performance when linking with hwloc

[jlpeyton]

Summary

When linking with libhwloc, the ORDERED EPCC test slows down on big machines (> 48 cores). Performance analysis showed that a cache thrash
was occurring and this padding helps alleviate the problem.

Also, inside the main spin-wait loop in kmp_wait_release.h, we can eliminate the references to the global shared variables by instead creating a local variable, oversubscribed and instead checking that.

[AndreyChurbanov]

LGTM

openmp/trunk/runtime/src/kmp.h

union shared_info {
         dispatch_shared_info64_t  s64;
     } u;
     volatile kmp_uint32     buffer_index;
 #if OMP_45_ENABLED
     volatile kmp_int32      doacross_buf_idx;  // teamwise index
     volatile kmp_uint32    *doacross_flags;    // shared array of iteration flags (0/1)
     kmp_int32               doacross_num_done; // count finished threads
 #endif
+#if KMP_USE_HWLOC
+    // When linking with libhwloc, the ORDERED EPCC test slows down on big
+    // machines (> 48 cores). Performance analysis showed that a cache thrash
+    // was occurring and this padding helps alleviate the problem.
+    char padding[64];
+#endif
 } dispatch_shared_info_t;
 
 typedef struct kmp_disp {
     /* Vector for ORDERED SECTION */
     void (*th_deo_fcn)( int * gtid, int * cid, ident_t *);
     /* Vector for END ORDERED SECTION */
     void (*th_dxo_fcn)( int * gtid, int * cid, ident_t *);
 
#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
     kmp_r_sched_t            t_sched;        // run-time schedule for the team
 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED
     int                      t_first_place;  // first & last place in parent thread's partition.
     int                      t_last_place;   // Restore these values to master after par region.
 #endif // OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED
     int t_size_changed; // team size was changed?: 0: no, 1: yes, -1: changed via omp_set_num_threads() call
 
     // Read/write by workers as well -----------------------------------------------------------------------
-#if KMP_ARCH_X86 || KMP_ARCH_X86_64
+#if (KMP_ARCH_X86 || KMP_ARCH_X86_64) && !KMP_USE_HWLOC
     // Using CACHE_LINE=64 reduces memory footprint, but causes a big perf regression of epcc 'parallel'
     // and 'barrier' on fxe256lin01. This extra padding serves to fix the performance of epcc 'parallel'
     // and 'barrier' when CACHE_LINE=64. TODO: investigate more and get rid if this padding.
     char dummy_padding[1024];
 #endif
     KMP_ALIGN_CACHE kmp_internal_control_t *t_control_stack_top;  // internal control stack for additional nested teams.
                                                    // for SERIALIZED teams nested 2 or more levels deep
 #if OMP_40_ENABLED

openmp/trunk/runtime/src/kmp_dispatch.cpp

union shared_info_tmpl {
         dispatch_shared_info64_t               s64;
     } u;
     volatile kmp_uint32     buffer_index;
 #if OMP_45_ENABLED
     volatile kmp_int32      doacross_buf_idx;  // teamwise index
     kmp_uint32             *doacross_flags;    // array of iteration flags (0/1)
     kmp_int32               doacross_num_done; // count finished threads
 #endif
+#if KMP_USE_HWLOC
+    // When linking with libhwloc, the ORDERED EPCC test slowsdown on big
+    // machines (> 48 cores). Performance analysis showed that a cache thrash
+    // was occurring and this padding helps alleviate the problem.
+    char padding[64];
+#endif
 };
 
 /* ------------------------------------------------------------------------ */
 /* ------------------------------------------------------------------------ */
 
 #undef USE_TEST_LOCKS
 
 // test_then_add template (general template should NOT be used)

openmp/trunk/runtime/src/kmp_wait_release.h

__kmp_wait_template(kmp_info_t *this_thr, C *flag, int final_spin
                     USE_ITT_BUILD_ARG(void * itt_sync_obj) )
 {
     // NOTE: We may not belong to a team at this point.
     volatile typename C::flag_t *spin = flag->get();
     kmp_uint32 spins;
     kmp_uint32 hibernate;
     int th_gtid;
     int tasks_completed = FALSE;
+    int oversubscribed;
 
     KMP_FSYNC_SPIN_INIT(spin, NULL);
     if (flag->done_check()) {
         KMP_FSYNC_SPIN_ACQUIRED(spin);
         return;
     }
     th_gtid = this_thr->th.th_info.ds.ds_gtid;
     KA_TRACE(20, ("__kmp_wait_sleep: T#%d waiting for flag(%p)\n", th_gtid, flag));
#endif /* KMP_ADJUST_BLOCKTIME */
 
         // Add in the current time value.
         hibernate += TCR_4(__kmp_global.g.g_time.dt.t_value);
         KF_TRACE(20, ("__kmp_wait_sleep: T#%d now=%d, hibernate=%d, intervals=%d\n",
                       th_gtid, __kmp_global.g.g_time.dt.t_value, hibernate,
                       hibernate - __kmp_global.g.g_time.dt.t_value));
     }
 
+    oversubscribed = (TCR_4(__kmp_nth) > __kmp_avail_proc);
     KMP_MB();
 
     // Main wait spin loop
     while (flag->notdone_check()) {
         int in_pool;
         kmp_task_team_t * task_team = NULL;
         if (__kmp_tasking_mode != tskm_immediate_exec) {
             task_team = this_thr->th.th_task_team;
while (flag->notdone_check()) {
         KMP_FSYNC_SPIN_PREPARE(spin);
         if (TCR_4(__kmp_global.g.g_done)) {
             if (__kmp_global.g.g_abort)
                 __kmp_abort_thread();
             break;
         }
 
         // If we are oversubscribed, or have waited a bit (and KMP_LIBRARY=throughput), then yield
-        KMP_YIELD(TCR_4(__kmp_nth) > __kmp_avail_proc);
+        KMP_YIELD(oversubscribed);
         // TODO: Should it be number of cores instead of thread contexts? Like:
         // KMP_YIELD(TCR_4(__kmp_nth) > __kmp_ncores);
         // Need performance improvement data to make the change...
         KMP_YIELD_SPIN(spins);
 
         // Check if this thread was transferred from a team
         // to the thread pool (or vice-versa) while spinning.
         in_pool = !!TCR_4(this_thr->th.th_in_pool);