Offer API for setting number of loop dispatch buffers

[jlpeyton]

Summary

The problem is the lack of dispatch buffers when thousands of loops with nowait, about 10 iterations each, are executed by hundreds of threads. We only have built-in 7 dispatch buffers, but there is a need in dozens or hundreds of buffers.

The problem can be fixed by setting KMP_MAX_DISP_BUF to bigger value. In order to give users same possibility I changed build-time control into run-time one, adding API just in case.

This change adds an environment variable KMP_DISP_NUM_BUFFERS and a new API function kmp_set_disp_num_buffers(int num_buffers).

The KMP_DISP_NUM_BUFFERS envirable works only before serial initialization, because during the serial initialization we already allocate buffers for the hot team, so it is too late to change the number of buffers later (or we need to reallocate buffers for all teams which sounds too complicated). The kmp_set_defaults() routine does not work for this envirable, because it calls serial initialization before reading the parameter string. So a new routine, kmp_set_disp_num_buffers(), is created so that it can set our internal global variable before the library initialization. If both the envirable and API used the envirable wins.

[tlwilmar]

LGTM

openmp/trunk/runtime/src/dllexports

__kmpc_proxy_task_completed_ooo     260
         __kmpc_doacross_init                261
         __kmpc_doacross_wait                262
         __kmpc_doacross_post                263
         __kmpc_doacross_fini                264
         __kmpc_taskloop                     266
     %endif
 %endif
 kmpc_aligned_malloc                         265
+kmpc_set_disp_num_buffers                   267
 
 # User API entry points that have both lower- and upper- case versions for Fortran.
 # Number for lowercase version is indicated.  Number for uppercase is obtained by adding 1000.
 # User API entry points are entry points that start with 'kmp_' or 'omp_'.
 
 omp_destroy_lock                            700
 omp_destroy_nest_lock                       701
 omp_get_dynamic                             702
%ifdef stub
         omp_target_is_present               885
         omp_target_memcpy                   886
         omp_target_memcpy_rect              887
         omp_target_associate_ptr            888
         omp_target_disassociate_ptr         889
     %endif
 %endif # OMP_41
 
+kmp_set_disp_num_buffers                    890
+
 %ifndef stub
     # Ordinals between 900 and 999 are reserved
 
     # Ordinals between 1000 and 1999 are reserved
     # for user-callable uppercase Fortran entries.
 
 
     # ATOMIC entries

openmp/trunk/runtime/src/i18n/en_US.txt

 AffThrPlaceNonThreeLevel     "KMP_PLACE_THREADS ignored: only three-level topology is supported."
 AffGranTopGroup              "%1$s: granularity=%2$s is not supported with KMP_TOPOLOGY_METHOD=group. Using \"granularity=fine\"."
 AffGranGroupType             "%1$s: granularity=group is not supported with KMP_AFFINITY=%2$s. Using \"granularity=core\"."
 AffThrPlaceManySockets       "KMP_PLACE_THREADS ignored: too many sockets requested."
 AffThrPlaceDeprecated        "KMP_PLACE_THREADS \"o\" offset designator deprecated, please use @ prefix for offset value."
 AffUsingHwloc                "%1$s: Affinity capable, using hwloc."
 AffIgnoringHwloc             "%1$s: Ignoring hwloc mechanism."
 AffHwlocErrorOccurred        "%1$s: Hwloc failed in %2$s. Relying on internal affinity mechanisms."
+EnvSerialWarn                "%1$s must be set prior to OpenMP runtime library initialization; ignored."
 
 
 # --------------------------------------------------------------------------------------------------
 -*- HINTS -*-
 # --------------------------------------------------------------------------------------------------
 
 # Hints. Hint may be printed after a message. Usually it is longer explanation text or suggestion.
 # To maintain hint numbers (they are visible to customers), add new hints to the end.

openmp/trunk/runtime/src/include/41/omp.h.var

#   include <stdlib.h>
     extern int    __KAI_KMPC_CONVENTION  kmp_get_blocktime          (void);
     extern int    __KAI_KMPC_CONVENTION  kmp_get_library            (void);
     extern void   __KAI_KMPC_CONVENTION  kmp_set_blocktime          (int);
     extern void   __KAI_KMPC_CONVENTION  kmp_set_library            (int);
     extern void   __KAI_KMPC_CONVENTION  kmp_set_library_serial     (void);
     extern void   __KAI_KMPC_CONVENTION  kmp_set_library_turnaround (void);
     extern void   __KAI_KMPC_CONVENTION  kmp_set_library_throughput (void);
     extern void   __KAI_KMPC_CONVENTION  kmp_set_defaults           (char const *);
+    extern void   __KAI_KMPC_CONVENTION  kmp_set_disp_num_buffers   (int);
 
     /* Intel affinity API */
     typedef void * kmp_affinity_mask_t;
 
     extern int    __KAI_KMPC_CONVENTION  kmp_set_affinity             (kmp_affinity_mask_t *);
     extern int    __KAI_KMPC_CONVENTION  kmp_get_affinity             (kmp_affinity_mask_t *);
     extern int    __KAI_KMPC_CONVENTION  kmp_get_affinity_max_proc    (void);
     extern void   __KAI_KMPC_CONVENTION  kmp_create_affinity_mask     (kmp_affinity_mask_t *);

openmp/trunk/runtime/src/include/41/omp_lib.h.var

!       ***
           integer (kind=omp_integer_kind) kmp_get_blocktime
         end function kmp_get_blocktime
 
         function kmp_get_library() bind(c)
           import
           integer (kind=omp_integer_kind) kmp_get_library
         end function kmp_get_library
 
+        subroutine kmp_set_disp_num_buffers(num) bind(c)
+          import
+          integer (kind=omp_integer_kind), value :: num
+        end subroutine kmp_set_disp_num_buffers
+
         function kmp_set_affinity(mask) bind(c)
           import
           integer (kind=omp_integer_kind) kmp_set_affinity
           integer (kind=kmp_affinity_mask_kind) mask
         end function kmp_set_affinity
 
         function kmp_get_affinity(mask) bind(c)
           import

 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_set_library_turnaround
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_set_library_throughput
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_set_library
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_set_defaults
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_get_stacksize
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_get_stacksize_s
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_get_blocktime
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_get_library
+!DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_set_disp_num_buffers
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_set_affinity
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_get_affinity
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_get_affinity_max_proc
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_create_affinity_mask
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_destroy_affinity_mask
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_set_affinity_mask_proc
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_unset_affinity_mask_proc
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_get_affinity_mask_proc

 !$omp declare target(kmp_set_library_turnaround )
 !$omp declare target(kmp_set_library_throughput )
 !$omp declare target(kmp_set_library )
 !$omp declare target(kmp_set_defaults )
 !$omp declare target(kmp_get_stacksize )
 !$omp declare target(kmp_get_stacksize_s )
 !$omp declare target(kmp_get_blocktime )
 !$omp declare target(kmp_get_library )
+!$omp declare target(kmp_set_disp_num_buffers )
 !$omp declare target(kmp_set_affinity )
 !$omp declare target(kmp_get_affinity )
 !$omp declare target(kmp_get_affinity_max_proc )
 !$omp declare target(kmp_create_affinity_mask )
 !$omp declare target(kmp_destroy_affinity_mask )
 !$omp declare target(kmp_set_affinity_mask_proc )
 !$omp declare target(kmp_unset_affinity_mask_proc )
 !$omp declare target(kmp_get_affinity_mask_proc )

openmp/trunk/runtime/src/include/41/omp_lib.f90.var

!         ***
             integer (kind=omp_integer_kind) kmp_get_blocktime
           end function kmp_get_blocktime
 
           function kmp_get_library() bind(c)
             use omp_lib_kinds
             integer (kind=omp_integer_kind) kmp_get_library
           end function kmp_get_library
 
+          subroutine kmp_set_disp_num_buffers(num) bind(c)
+            use omp_lib_kinds
+            integer (kind=omp_integer_kind), value :: num
+          end subroutine kmp_set_disp_num_buffers
+
           function kmp_set_affinity(mask) bind(c)
             use omp_lib_kinds
             integer (kind=omp_integer_kind) kmp_set_affinity
             integer (kind=kmp_affinity_mask_kind) mask
           end function kmp_set_affinity
 
           function kmp_get_affinity(mask) bind(c)
             use omp_lib_kinds

openmp/trunk/runtime/src/kmp.h

 #define KMP_MIN_INIT_WAIT       1
 #define KMP_MAX_INIT_WAIT       (INT_MAX/2)
 #define KMP_DEFAULT_INIT_WAIT   2048U
 
 #define KMP_MIN_NEXT_WAIT       1
 #define KMP_MAX_NEXT_WAIT       (INT_MAX/2)
 #define KMP_DEFAULT_NEXT_WAIT   1024U
 
-// max possible dynamic loops in concurrent execution per team
-#define KMP_MAX_DISP_BUF        7
+#define KMP_DFLT_DISP_NUM_BUFF  7
 #define KMP_MAX_ORDERED         8
 
 #define KMP_MAX_FIELDS          32
 
 #define KMP_MAX_BRANCH_BITS     31
 
 #define KMP_MAX_ACTIVE_LEVELS_LIMIT INT_MAX
 

 extern int        __kmp_foreign_tp;     /* If true, separate TP var for each foreign thread */
 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
 extern int        __kmp_inherit_fp_control; /* copy fp creg(s) parent->workers at fork */
 extern kmp_int16  __kmp_init_x87_fpu_control_word; /* init thread's FP control reg */
 extern kmp_uint32 __kmp_init_mxcsr;      /* init thread's mxscr */
 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
 
 extern int        __kmp_dflt_max_active_levels; /* max_active_levels for nested parallelism enabled by default a la OMP_MAX_ACTIVE_LEVELS */
+extern int        __kmp_dispatch_num_buffers; /* max possible dynamic loops in concurrent execution per team */
 #if KMP_NESTED_HOT_TEAMS
 extern int        __kmp_hot_teams_mode;
 extern int        __kmp_hot_teams_max_level;
 #endif
 
 # if KMP_OS_LINUX
 extern enum clock_function_type __kmp_clock_function;
 extern int __kmp_clock_function_param;

 KMP_EXPORT int  KMPC_CONVENTION kmpc_set_affinity_mask_proc(int, kmp_affinity_mask_t *);
 KMP_EXPORT int  KMPC_CONVENTION kmpc_unset_affinity_mask_proc(int, kmp_affinity_mask_t *);
 KMP_EXPORT int  KMPC_CONVENTION kmpc_get_affinity_mask_proc(int, kmp_affinity_mask_t *);
 
 KMP_EXPORT void KMPC_CONVENTION kmpc_set_stacksize(int);
 KMP_EXPORT void KMPC_CONVENTION kmpc_set_stacksize_s(size_t);
 KMP_EXPORT void KMPC_CONVENTION kmpc_set_library(int);
 KMP_EXPORT void KMPC_CONVENTION kmpc_set_defaults(char const *);
+KMP_EXPORT void KMPC_CONVENTION kmpc_set_disp_num_buffers(int);
 
 #ifdef __cplusplus
 }
 #endif
 
 #endif /* KMP_H */
 

openmp/trunk/runtime/src/kmp_csupport.c

 
 void
 kmpc_set_defaults( char const * str )
 {
     // __kmp_aux_set_defaults initializes the library if needed
     __kmp_aux_set_defaults( str, KMP_STRLEN( str ) );
 }
 
+void
+kmpc_set_disp_num_buffers( int arg )
+{
+    // ignore after initialization because some teams have already
+    // allocated dispatch buffers
+    if( __kmp_init_serial == 0 && arg > 0 )
+        __kmp_dispatch_num_buffers = arg;
+}
+
 int
 kmpc_set_affinity_mask_proc( int proc, void **mask )
 {
 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
     return -1;
 #else
     if ( ! TCR_4(__kmp_init_middle) ) {
         __kmp_middle_initialize();
__kmpc_doacross_init(ident_t *loc, int gtid, int num_dims, struct kmp_dim * dims)
     KMP_DEBUG_ASSERT(num_dims > 0);
 
     if( team->t.t_serialized ) {
         KA_TRACE(20,("__kmpc_doacross_init() exit: serialized team\n"));
         return; // no dependencies if team is serialized
     }
     KMP_DEBUG_ASSERT(team->t.t_nproc > 1);
     idx = pr_buf->th_doacross_buf_idx++; // Increment index of shared buffer for the next loop
-    sh_buf = &team->t.t_disp_buffer[idx % KMP_MAX_DISP_BUF];
+    sh_buf = &team->t.t_disp_buffer[idx % __kmp_dispatch_num_buffers];
 
     // Save bounds info into allocated private buffer
     KMP_DEBUG_ASSERT(pr_buf->th_doacross_info == NULL);
     pr_buf->th_doacross_info =
         (kmp_int64*)__kmp_thread_malloc(th, sizeof(kmp_int64)*(4 * num_dims + 1));
     KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
     pr_buf->th_doacross_info[0] = (kmp_int64)num_dims; // first element is number of dimensions
     // Save also address of num_done in order to access it later without knowing the buffer index
if( dims[0].st == 1 ) { // most common case
         KMP_DEBUG_ASSERT(dims[0].lo > dims[0].up);
         trace_count = (kmp_uint64)(dims[0].lo - dims[0].up) / (-dims[0].st) + 1;
     }
     for( j = 1; j < num_dims; ++j ) {
         trace_count *= pr_buf->th_doacross_info[4 * j + 1]; // use kept ranges
     }
     KMP_DEBUG_ASSERT(trace_count > 0);
 
-    // Check if shared buffer is not occupied by other loop (idx - KMP_MAX_DISP_BUF)
+    // Check if shared buffer is not occupied by other loop (idx - __kmp_dispatch_num_buffers)
     if( idx != sh_buf->doacross_buf_idx ) {
         // Shared buffer is occupied, wait for it to be free
         __kmp_wait_yield_4( (kmp_uint32*)&sh_buf->doacross_buf_idx, idx, __kmp_eq_4, NULL );
     }
     // Check if we are the first thread. After the CAS the first thread gets 0,
     // others get 1 if initialization is in progress, allocated pointer otherwise.
     flags = (kmp_uint32*)KMP_COMPARE_AND_STORE_RET64(
         (kmp_int64*)&sh_buf->doacross_flags,NULL,(kmp_int64)1);
__kmpc_doacross_fini(ident_t *loc, int gtid)
     if( team->t.t_serialized ) {
         KA_TRACE(20,("__kmpc_doacross_fini() exit: serialized team %p\n", team));
         return; // nothing to do
     }
     num_done = KMP_TEST_THEN_INC64((kmp_int64*)pr_buf->th_doacross_info[1]) + 1;
     if( num_done == th->th.th_team_nproc ) {
         // we are the last thread, need to free shared resources
         int idx = pr_buf->th_doacross_buf_idx - 1;
-        dispatch_shared_info_t *sh_buf = &team->t.t_disp_buffer[idx % KMP_MAX_DISP_BUF];
+        dispatch_shared_info_t *sh_buf = &team->t.t_disp_buffer[idx % __kmp_dispatch_num_buffers];
         KMP_DEBUG_ASSERT(pr_buf->th_doacross_info[1] == (kmp_int64)&sh_buf->doacross_num_done);
         KMP_DEBUG_ASSERT(num_done == (kmp_int64)sh_buf->doacross_num_done);
         KMP_DEBUG_ASSERT(idx == sh_buf->doacross_buf_idx);
         __kmp_thread_free(th, (void*)sh_buf->doacross_flags);
         sh_buf->doacross_flags = NULL;
         sh_buf->doacross_num_done = 0;
-        sh_buf->doacross_buf_idx += KMP_MAX_DISP_BUF; // free buffer for future re-use
+        sh_buf->doacross_buf_idx += __kmp_dispatch_num_buffers; // free buffer for future re-use
     }
     // free private resources (need to keep buffer index forever)
     __kmp_thread_free(th, (void*)pr_buf->th_doacross_info);
     pr_buf->th_doacross_info = NULL;
     KA_TRACE(20,("__kmpc_doacross_fini() exit: T#%d\n", gtid));
 }
 #endif
 
 // end of file //
 

openmp/trunk/runtime/src/kmp_dispatch.cpp

#endif
     } else {
         KMP_DEBUG_ASSERT( th->th.th_dispatch ==
                 &th->th.th_team->t.t_dispatch[th->th.th_info.ds.ds_tid] );
 
         my_buffer_index = th->th.th_dispatch->th_disp_index ++;
 
         /* What happens when number of threads changes, need to resize buffer? */
         pr = reinterpret_cast< dispatch_private_info_template< T >  * >
-            ( &th -> th.th_dispatch -> th_disp_buffer[ my_buffer_index % KMP_MAX_DISP_BUF ] );
+            ( &th -> th.th_dispatch -> th_disp_buffer[ my_buffer_index % __kmp_dispatch_num_buffers ] );
         sh = reinterpret_cast< dispatch_shared_info_template< UT > volatile * >
-            ( &team -> t.t_disp_buffer[ my_buffer_index % KMP_MAX_DISP_BUF ] );
+            ( &team -> t.t_disp_buffer[ my_buffer_index % __kmp_dispatch_num_buffers ] );
     }
 
     /* Currently just ignore the monotonic and non-monotonic modifiers (the compiler isn't producing them
      * yet anyway).
      * When it is we'll want to look at them somewhere here and use that information to add to our
      * schedule choice. We shouldn't need to pass them on, they merely affect which schedule we can
      * legally choose for various dynamic cases. (In paritcular, whether or not a stealing scheme is legal).
      */
#endif
 
                 /* TODO replace with general release procedure? */
                 if ( pr->ordered ) {
                     sh->u.s.ordered_iteration = 0;
                 }
 
                 KMP_MB();       /* Flush all pending memory write invalidates.  */
 
-                sh -> buffer_index += KMP_MAX_DISP_BUF;
+                sh -> buffer_index += __kmp_dispatch_num_buffers;
                 KD_TRACE(100, ("__kmp_dispatch_next: T#%d change buffer_index:%d\n",
                                 gtid, sh->buffer_index) );
 
                 KMP_MB();       /* Flush all pending memory write invalidates.  */
 
             } // if
             if ( __kmp_env_consistency_check ) {
                 if ( pr->pushed_ws != ct_none ) {

openmp/trunk/runtime/src/kmp_ftn_entry.h

FTN_GET_LIBRARY (void)
     #else
         if ( ! __kmp_init_serial ) {
             __kmp_serial_initialize();
         }
         return ((int) __kmp_library);
     #endif
 }
 
+void FTN_STDCALL
+FTN_SET_DISP_NUM_BUFFERS( int KMP_DEREF arg )
+{
+    #ifdef KMP_STUB
+        ; // empty routine
+    #else
+        // ignore after initialization because some teams have already
+        // allocated dispatch buffers
+        if( __kmp_init_serial == 0 && (KMP_DEREF arg) > 0 )
+            __kmp_dispatch_num_buffers = KMP_DEREF arg;
+    #endif
+}
+
 int FTN_STDCALL
 FTN_SET_AFFINITY( void **mask )
 {
     #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
         return -1;
     #else
         if ( ! TCR_4(__kmp_init_middle) ) {
             __kmp_middle_initialize();

openmp/trunk/runtime/src/kmp_ftn_os.h

#if KMP_FTN_ENTRIES == KMP_FTN_PLAIN
     #define FTN_SET_BLOCKTIME                    kmp_set_blocktime
     #define FTN_GET_BLOCKTIME                    kmp_get_blocktime
     #define FTN_SET_LIBRARY_SERIAL               kmp_set_library_serial
     #define FTN_SET_LIBRARY_TURNAROUND           kmp_set_library_turnaround
     #define FTN_SET_LIBRARY_THROUGHPUT           kmp_set_library_throughput
     #define FTN_SET_LIBRARY                      kmp_set_library
     #define FTN_GET_LIBRARY                      kmp_get_library
     #define FTN_SET_DEFAULTS                     kmp_set_defaults
+    #define FTN_SET_DISP_NUM_BUFFERS             kmp_set_disp_num_buffers
     #define FTN_SET_AFFINITY                     kmp_set_affinity
     #define FTN_GET_AFFINITY                     kmp_get_affinity
     #define FTN_GET_AFFINITY_MAX_PROC            kmp_get_affinity_max_proc
     #define FTN_CREATE_AFFINITY_MASK             kmp_create_affinity_mask
     #define FTN_DESTROY_AFFINITY_MASK            kmp_destroy_affinity_mask
     #define FTN_SET_AFFINITY_MASK_PROC           kmp_set_affinity_mask_proc
     #define FTN_UNSET_AFFINITY_MASK_PROC         kmp_unset_affinity_mask_proc
     #define FTN_GET_AFFINITY_MASK_PROC           kmp_get_affinity_mask_proc
#if KMP_FTN_ENTRIES == KMP_FTN_APPEND
     #define FTN_SET_BLOCKTIME                    kmp_set_blocktime_
     #define FTN_GET_BLOCKTIME                    kmp_get_blocktime_
     #define FTN_SET_LIBRARY_SERIAL               kmp_set_library_serial_
     #define FTN_SET_LIBRARY_TURNAROUND           kmp_set_library_turnaround_
     #define FTN_SET_LIBRARY_THROUGHPUT           kmp_set_library_throughput_
     #define FTN_SET_LIBRARY                      kmp_set_library_
     #define FTN_GET_LIBRARY                      kmp_get_library_
     #define FTN_SET_DEFAULTS                     kmp_set_defaults_
+    #define FTN_SET_DISP_NUM_BUFFERS             kmp_set_disp_num_buffers_
     #define FTN_SET_AFFINITY                     kmp_set_affinity_
     #define FTN_GET_AFFINITY                     kmp_get_affinity_
     #define FTN_GET_AFFINITY_MAX_PROC            kmp_get_affinity_max_proc_
     #define FTN_CREATE_AFFINITY_MASK             kmp_create_affinity_mask_
     #define FTN_DESTROY_AFFINITY_MASK            kmp_destroy_affinity_mask_
     #define FTN_SET_AFFINITY_MASK_PROC           kmp_set_affinity_mask_proc_
     #define FTN_UNSET_AFFINITY_MASK_PROC         kmp_unset_affinity_mask_proc_
     #define FTN_GET_AFFINITY_MASK_PROC           kmp_get_affinity_mask_proc_
#if KMP_FTN_ENTRIES == KMP_FTN_UPPER
     #define FTN_SET_BLOCKTIME                    KMP_SET_BLOCKTIME
     #define FTN_GET_BLOCKTIME                    KMP_GET_BLOCKTIME
     #define FTN_SET_LIBRARY_SERIAL               KMP_SET_LIBRARY_SERIAL
     #define FTN_SET_LIBRARY_TURNAROUND           KMP_SET_LIBRARY_TURNAROUND
     #define FTN_SET_LIBRARY_THROUGHPUT           KMP_SET_LIBRARY_THROUGHPUT
     #define FTN_SET_LIBRARY                      KMP_SET_LIBRARY
     #define FTN_GET_LIBRARY                      KMP_GET_LIBRARY
     #define FTN_SET_DEFAULTS                     KMP_SET_DEFAULTS
+    #define FTN_SET_DISP_NUM_BUFFERS             KMP_SET_DISP_NUM_BUFFERS
     #define FTN_SET_AFFINITY                     KMP_SET_AFFINITY
     #define FTN_GET_AFFINITY                     KMP_GET_AFFINITY
     #define FTN_GET_AFFINITY_MAX_PROC            KMP_GET_AFFINITY_MAX_PROC
     #define FTN_CREATE_AFFINITY_MASK             KMP_CREATE_AFFINITY_MASK
     #define FTN_DESTROY_AFFINITY_MASK            KMP_DESTROY_AFFINITY_MASK
     #define FTN_SET_AFFINITY_MASK_PROC           KMP_SET_AFFINITY_MASK_PROC
     #define FTN_UNSET_AFFINITY_MASK_PROC         KMP_UNSET_AFFINITY_MASK_PROC
     #define FTN_GET_AFFINITY_MASK_PROC           KMP_GET_AFFINITY_MASK_PROC
#if KMP_FTN_ENTRIES == KMP_FTN_UAPPEND
     #define FTN_SET_BLOCKTIME                    KMP_SET_BLOCKTIME_
     #define FTN_GET_BLOCKTIME                    KMP_GET_BLOCKTIME_
     #define FTN_SET_LIBRARY_SERIAL               KMP_SET_LIBRARY_SERIAL_
     #define FTN_SET_LIBRARY_TURNAROUND           KMP_SET_LIBRARY_TURNAROUND_
     #define FTN_SET_LIBRARY_THROUGHPUT           KMP_SET_LIBRARY_THROUGHPUT_
     #define FTN_SET_LIBRARY                      KMP_SET_LIBRARY_
     #define FTN_GET_LIBRARY                      KMP_GET_LIBRARY_
     #define FTN_SET_DEFAULTS                     KMP_SET_DEFAULTS_
+    #define FTN_SET_DISP_NUM_BUFFERS             KMP_SET_DISP_NUM_BUFFERS_
     #define FTN_SET_AFFINITY                     KMP_SET_AFFINITY_
     #define FTN_GET_AFFINITY                     KMP_GET_AFFINITY_
     #define FTN_GET_AFFINITY_MAX_PROC            KMP_GET_AFFINITY_MAX_PROC_
     #define FTN_CREATE_AFFINITY_MASK             KMP_CREATE_AFFINITY_MASK_
     #define FTN_DESTROY_AFFINITY_MASK            KMP_DESTROY_AFFINITY_MASK_
     #define FTN_SET_AFFINITY_MASK_PROC           KMP_SET_AFFINITY_MASK_PROC_
     #define FTN_UNSET_AFFINITY_MASK_PROC         KMP_UNSET_AFFINITY_MASK_PROC_
     #define FTN_GET_AFFINITY_MASK_PROC           KMP_GET_AFFINITY_MASK_PROC_

openmp/trunk/runtime/src/kmp_global.c

 int           __kmp_sys_max_nth = KMP_MAX_NTH;
 int               __kmp_max_nth = 0;
 int      __kmp_threads_capacity = 0;
 int         __kmp_dflt_team_nth = 0;
 int      __kmp_dflt_team_nth_ub = 0;
 int           __kmp_tp_capacity = 0;
 int             __kmp_tp_cached = 0;
 int           __kmp_dflt_nested = FALSE;
+int  __kmp_dispatch_num_buffers = KMP_DFLT_DISP_NUM_BUFF;
 int __kmp_dflt_max_active_levels = KMP_MAX_ACTIVE_LEVELS_LIMIT; /* max_active_levels limit */
 #if KMP_NESTED_HOT_TEAMS
 int __kmp_hot_teams_mode         = 0; /* 0 - free extra threads when reduced */
                                       /* 1 - keep extra threads when reduced */
 int __kmp_hot_teams_max_level    = 1; /* nesting level of hot teams */
 #endif
 enum library_type __kmp_library = library_none;
 enum sched_type     __kmp_sched = kmp_sch_default;  /* scheduling method for runtime scheduling */

openmp/trunk/runtime/src/kmp_runtime.c

 /*
  * Print out the storage map for the major kmp_team_t team data structures
  * that are allocated together.
  */
 
 static void
 __kmp_print_team_storage_map( const char *header, kmp_team_t *team, int team_id, int num_thr )
 {
-    int num_disp_buff = team->t.t_max_nproc > 1 ? KMP_MAX_DISP_BUF : 2;
+    int num_disp_buff = team->t.t_max_nproc > 1 ? __kmp_dispatch_num_buffers : 2;
     __kmp_print_storage_map_gtid( -1, team, team + 1, sizeof(kmp_team_t), "%s_%d",
                              header, team_id );
 
     __kmp_print_storage_map_gtid( -1, &team->t.t_bar[0], &team->t.t_bar[bs_last_barrier],
                              sizeof(kmp_balign_team_t) * bs_last_barrier, "%s_%d.t_bar", header, team_id );
 
 
     __kmp_print_storage_map_gtid( -1, &team->t.t_bar[bs_plain_barrier], &team->t.t_bar[bs_plain_barrier+1],
if( !realloc || argc > team->t.t_max_argc ) {
         }
     }
 }
 
 static void
 __kmp_allocate_team_arrays(kmp_team_t *team, int max_nth)
 {
     int i;
-    int num_disp_buff = max_nth > 1 ? KMP_MAX_DISP_BUF : 2;
+    int num_disp_buff = max_nth > 1 ? __kmp_dispatch_num_buffers : 2;
     team->t.t_threads = (kmp_info_t**) __kmp_allocate( sizeof(kmp_info_t*) * max_nth );
     team->t.t_disp_buffer = (dispatch_shared_info_t*)
         __kmp_allocate( sizeof(dispatch_shared_info_t) * num_disp_buff );
     team->t.t_dispatch = (kmp_disp_t*) __kmp_allocate( sizeof(kmp_disp_t) * max_nth );
     team->t.t_implicit_task_taskdata = (kmp_taskdata_t*) __kmp_allocate( sizeof(kmp_taskdata_t) * max_nth );
     team->t.t_max_nproc = max_nth;
 
     /* setup dispatch buffers */
#endif
 
     /* Initialize dynamic dispatch */
     {
         volatile kmp_disp_t *dispatch = this_thr->th.th_dispatch;
         /*
          * Use team max_nproc since this will never change for the team.
          */
         size_t disp_size = sizeof( dispatch_private_info_t ) *
-            ( team->t.t_max_nproc == 1 ? 1 : KMP_MAX_DISP_BUF );
+            ( team->t.t_max_nproc == 1 ? 1 : __kmp_dispatch_num_buffers );
         KD_TRACE( 10, ("__kmp_initialize_info: T#%d max_nproc: %d\n", gtid, team->t.t_max_nproc ) );
         KMP_ASSERT( dispatch );
         KMP_DEBUG_ASSERT( team->t.t_dispatch );
         KMP_DEBUG_ASSERT( dispatch == &team->t.t_dispatch[ tid ] );
 
         dispatch->th_disp_index = 0;
 #if OMP_41_ENABLED
         dispatch->th_doacross_buf_idx = 0;
 #endif
         if( ! dispatch->th_disp_buffer )  {
             dispatch->th_disp_buffer = (dispatch_private_info_t *) __kmp_allocate( disp_size );
 
             if ( __kmp_storage_map ) {
                 __kmp_print_storage_map_gtid( gtid, &dispatch->th_disp_buffer[ 0 ],
-                                         &dispatch->th_disp_buffer[ team->t.t_max_nproc == 1 ? 1 : KMP_MAX_DISP_BUF ],
+                                         &dispatch->th_disp_buffer[ team->t.t_max_nproc == 1 ? 1 : __kmp_dispatch_num_buffers ],
                                          disp_size, "th_%d.th_dispatch.th_disp_buffer "
                                          "(team_%d.t_dispatch[%d].th_disp_buffer)",
                                          gtid, team->t.t_id, gtid );
             }
         } else {
             memset( & dispatch->th_disp_buffer[0], '\0', disp_size );
         }
 
#endif /* KMP_DEBUG */
 
     team->t.t_construct = 0;          /* no single directives seen yet */
     team->t.t_ordered.dt.t_value = 0; /* thread 0 enters the ordered section first */
 
     /* Reset the identifiers on the dispatch buffer */
     KMP_DEBUG_ASSERT( team->t.t_disp_buffer );
     if ( team->t.t_max_nproc > 1 ) {
         int i;
-        for (i = 0; i <  KMP_MAX_DISP_BUF; ++i) {
+        for (i = 0; i <  __kmp_dispatch_num_buffers; ++i) {
             team->t.t_disp_buffer[ i ].buffer_index = i;
 #if OMP_41_ENABLED
             team->t.t_disp_buffer[i].doacross_buf_idx = i;
 #endif
         }
     } else {
         team->t.t_disp_buffer[ 0 ].buffer_index = 0;
 #if OMP_41_ENABLED

openmp/trunk/runtime/src/kmp_settings.c

 } // __kmp_stg_parse_max_task_priority
 
 static void
 __kmp_stg_print_max_task_priority(kmp_str_buf_t *buffer, char const *name, void *data) {
     __kmp_stg_print_int(buffer, name, __kmp_max_task_priority);
 } // __kmp_stg_print_max_task_priority
 #endif // OMP_41_ENABLED
 
+// -------------------------------------------------------------------------------------------------
+// KMP_DISP_NUM_BUFFERS
+// -------------------------------------------------------------------------------------------------
+static void
+__kmp_stg_parse_disp_buffers( char const * name, char const * value, void * data ) {
+    if ( TCR_4(__kmp_init_serial) ) {
+        KMP_WARNING( EnvSerialWarn, name );
+        return;
+    }   // read value before serial initialization only
+    __kmp_stg_parse_int( name, value, 1, KMP_MAX_NTH, & __kmp_dispatch_num_buffers );
+} // __kmp_stg_parse_disp_buffers
+
+static void
+__kmp_stg_print_disp_buffers( kmp_str_buf_t * buffer, char const * name, void * data ) {
+    __kmp_stg_print_int( buffer, name, __kmp_dispatch_num_buffers );
+} // __kmp_stg_print_disp_buffers
+
 #if KMP_NESTED_HOT_TEAMS
 // -------------------------------------------------------------------------------------------------
 // KMP_HOT_TEAMS_MAX_LEVEL, KMP_HOT_TEAMS_MODE
 // -------------------------------------------------------------------------------------------------
 
 static void
 __kmp_stg_parse_hot_teams_level( char const * name, char const * value, void * data ) {
     if ( TCR_4(__kmp_init_parallel) ) {
static kmp_setting_t __kmp_stg_table[] = {
     { "KMP_TASKING",                       __kmp_stg_parse_tasking,            __kmp_stg_print_tasking,            NULL, 0, 0 },
     { "KMP_TASK_STEALING_CONSTRAINT",      __kmp_stg_parse_task_stealing,      __kmp_stg_print_task_stealing,      NULL, 0, 0 },
     { "OMP_MAX_ACTIVE_LEVELS",             __kmp_stg_parse_max_active_levels,  __kmp_stg_print_max_active_levels,  NULL, 0, 0 },
 #if OMP_41_ENABLED
     { "OMP_MAX_TASK_PRIORITY",             __kmp_stg_parse_max_task_priority,  __kmp_stg_print_max_task_priority,  NULL, 0, 0 },
 #endif
     { "OMP_THREAD_LIMIT",                  __kmp_stg_parse_all_threads,        __kmp_stg_print_all_threads,        NULL, 0, 0 },
     { "OMP_WAIT_POLICY",                   __kmp_stg_parse_wait_policy,        __kmp_stg_print_wait_policy,        NULL, 0, 0 },
+    { "KMP_DISP_NUM_BUFFERS",              __kmp_stg_parse_disp_buffers,       __kmp_stg_print_disp_buffers,       NULL, 0, 0 },
 #if KMP_NESTED_HOT_TEAMS
     { "KMP_HOT_TEAMS_MAX_LEVEL",           __kmp_stg_parse_hot_teams_level,    __kmp_stg_print_hot_teams_level,    NULL, 0, 0 },
     { "KMP_HOT_TEAMS_MODE",                __kmp_stg_parse_hot_teams_mode,     __kmp_stg_print_hot_teams_mode,     NULL, 0, 0 },
 #endif // KMP_NESTED_HOT_TEAMS
 
 #if KMP_HANDLE_SIGNALS
     { "KMP_HANDLE_SIGNALS",                __kmp_stg_parse_handle_signals,     __kmp_stg_print_handle_signals,     NULL, 0, 0 },
 #endif

openmp/trunk/runtime/src/kmp_stub.c

 #define omp_set_num_threads          ompc_set_num_threads
 #define omp_set_dynamic              ompc_set_dynamic
 #define omp_set_nested               ompc_set_nested
 #define kmp_set_stacksize            kmpc_set_stacksize
 #define kmp_set_stacksize_s          kmpc_set_stacksize_s
 #define kmp_set_blocktime            kmpc_set_blocktime
 #define kmp_set_library              kmpc_set_library
 #define kmp_set_defaults             kmpc_set_defaults
+#define kmp_set_disp_num_buffers     kmpc_set_disp_num_buffers
 #define kmp_malloc                   kmpc_malloc
 #define kmp_aligned_malloc           kmpc_aligned_malloc
 #define kmp_calloc                   kmpc_calloc
 #define kmp_realloc                  kmpc_realloc
 #define kmp_free                     kmpc_free
 
 static double frequency = 0.0;
 

 int kmpc_get_affinity_mask_proc( int proc, void **mask ) { i; return -1; }
 
 /* kmp API functions */
 void kmp_set_stacksize( omp_int_t arg )   { i; __kmps_set_stacksize( arg ); }
 void kmp_set_stacksize_s( size_t arg )    { i; __kmps_set_stacksize( arg ); }
 void kmp_set_blocktime( omp_int_t arg )   { i; __kmps_set_blocktime( arg ); }
 void kmp_set_library( omp_int_t arg )     { i; __kmps_set_library( arg ); }
 void kmp_set_defaults( char const * str ) { i; }
+void kmp_set_disp_num_buffers( omp_int_t arg ) { i; }
 
 /* KMP memory management functions. */
 void * kmp_malloc( size_t size )                 { i; return malloc( size ); }
 void * kmp_aligned_malloc( size_t sz, size_t a ) {
     i;
 #if KMP_OS_WINDOWS
     errno = ENOSYS; // not supported
     return NULL;    // no standard aligned allocator on Windows (pre - C11)

openmp/trunk/runtime/test/env/kmp_set_dispatch_buf.c

+// RUN: %libomp-compile && env KMP_DISP_NUM_BUFFERS=0 %libomp-run
+// RUN: env KMP_DISP_NUM_BUFFERS=1 %libomp-run && env KMP_DISP_NUM_BUFFERS=3 %libomp-run
+// RUN: env KMP_DISP_NUM_BUFFERS=4 %libomp-run && env KMP_DISP_NUM_BUFFERS=7 %libomp-run
+// RUN: %libomp-compile -DMY_SCHEDULE=guided && env KMP_DISP_NUM_BUFFERS=1 %libomp-run
+// RUN: env KMP_DISP_NUM_BUFFERS=3 %libomp-run && env KMP_DISP_NUM_BUFFERS=4 %libomp-run
+// RUN: env KMP_DISP_NUM_BUFFERS=7 %libomp-run
+#include <stdio.h>
+#include <omp.h>
+#include <stdlib.h>
+#include <limits.h>
+#include "omp_testsuite.h"
+
+#define INCR 7
+#define MY_MAX 200
+#define MY_MIN -200
+#define NUM_LOOPS 100
+#ifndef MY_SCHEDULE
+# define MY_SCHEDULE dynamic
+#endif
+
+int a, b, a_known_value, b_known_value;
+
+int test_kmp_set_disp_num_buffers()
+{
+  int success = 1;
+  a = 0;
+  b = 0;
+  // run many small dynamic loops to stress the dispatch buffer system
+  #pragma omp parallel
+  {
+    int i,j;
+    for (j = 0; j < NUM_LOOPS; j++) {
+      #pragma omp for schedule(MY_SCHEDULE) nowait
+      for (i = MY_MIN; i < MY_MAX; i+=INCR) {
+        #pragma omp atomic
+        a++;
+      }
+      #pragma omp for schedule(MY_SCHEDULE) nowait
+      for (i = MY_MAX; i >= MY_MIN; i-=INCR) {
+        #pragma omp atomic
+        b++;
+      }
+    }
+  }
+  // detect failure
+  if (a != a_known_value || b != b_known_value) {
+    success = 0;
+    printf("a = %d (should be %d), b = %d (should be %d)\n", a, a_known_value,
+           b, b_known_value);
+  }
+  return success;
+}
+
+int main(int argc, char** argv)
+{
+  int i,j;
+  int num_failed=0;
+
+  // figure out the known values to compare with calculated result
+  a_known_value = 0;
+  b_known_value = 0;
+
+  for (j = 0; j < NUM_LOOPS; j++) {
+    for (i = MY_MIN; i < MY_MAX; i+=INCR)
+      a_known_value++;
+    for (i = MY_MAX; i >= MY_MIN; i-=INCR)
+      b_known_value++;
+  }
+
+  for(i = 0; i < REPETITIONS; i++) {
+    if(!test_kmp_set_disp_num_buffers()) {
+      num_failed++;
+    }
+  }
+  return num_failed;
+}

openmp/trunk/runtime/test/worksharing/for/kmp_set_dispatch_buf.c

+// RUN: %libomp-compile && %libomp-run 7
+// RUN: %libomp-run 0 && %libomp-run -1
+// RUN: %libomp-run 1 && %libomp-run 2 && %libomp-run 5
+// RUN: %libomp-compile -DMY_SCHEDULE=guided && %libomp-run 7
+// RUN: %libomp-run 1 && %libomp-run 2 && %libomp-run 5
+#include <stdio.h>
+#include <omp.h>
+#include <stdlib.h>
+#include <limits.h>
+#include "omp_testsuite.h"
+
+#define INCR 7
+#define MY_MAX 200
+#define MY_MIN -200
+#ifndef MY_SCHEDULE
+# define MY_SCHEDULE dynamic
+#endif
+
+int num_disp_buffers, num_loops;
+int a, b, a_known_value, b_known_value;
+
+int test_kmp_set_disp_num_buffers()
+{
+  int success = 1;
+  a = 0;
+  b = 0;
+  // run many small dynamic loops to stress the dispatch buffer system
+  #pragma omp parallel
+  {
+    int i,j;
+    for (j = 0; j < num_loops; j++) {
+      #pragma omp for schedule(MY_SCHEDULE) nowait
+      for (i = MY_MIN; i < MY_MAX; i+=INCR) {
+        #pragma omp atomic
+        a++;
+      }
+      #pragma omp for schedule(MY_SCHEDULE) nowait
+      for (i = MY_MAX; i >= MY_MIN; i-=INCR) {
+        #pragma omp atomic
+        b++;
+      }
+    }
+  }
+  // detect failure
+  if (a != a_known_value || b != b_known_value) {
+    success = 0;
+    printf("a = %d (should be %d), b = %d (should be %d)\n", a, a_known_value,
+           b, b_known_value);
+  }
+  return success;
+}
+
+int main(int argc, char** argv)
+{
+  int i,j;
+  int num_failed=0;
+
+  if (argc != 2) {
+    fprintf(stderr, "usage: %s num_disp_buffers\n", argv[0]);
+    exit(1);
+  }
+
+  // set the number of dispatch buffers
+  num_disp_buffers = atoi(argv[1]);
+  kmp_set_disp_num_buffers(num_disp_buffers);
+
+  // figure out the known values to compare with calculated result
+  a_known_value = 0;
+  b_known_value = 0;
+
+  // if specified to use bad num_disp_buffers set num_loops
+  // to something reasonable
+  if (num_disp_buffers <= 0)
+    num_loops = 10;
+  else
+    num_loops = num_disp_buffers*10;
+
+  for (j = 0; j < num_loops; j++) {
+    for (i = MY_MIN; i < MY_MAX; i+=INCR)
+      a_known_value++;
+    for (i = MY_MAX; i >= MY_MIN; i-=INCR)
+      b_known_value++;
+  }
+
+  for(i = 0; i < REPETITIONS; i++) {
+    if(!test_kmp_set_disp_num_buffers()) {
+      num_failed++;
+    }
+  }
+  return num_failed;
+}