Fix implementation of OMP_THREAD_LIMIT

[tlwilmar]

Summary

This change fixes the implementation of OMP_THREAD_LIMIT. The implementation of this previously was not restricted to a contention group (but it should be, according to the spec), and this is fixed here. A field is added to root thread to store a counter of the threads in the contention group. An extra check is added when reserving threads for a parallel region that checks this variable and compares to threadlimit-var, which is implemented as a new global variable, kmp_cg_max_nth. Associated settings changes were also made, and clean up of comments that referred to OMP_THREAD_LIMIT, but should refer to the new KMP_DEVICE_THREAD_LIMIT (added in an earlier patch).

[tlwilmar]

Added a test for OMP_THREAD_LIMIT env, api, and general functionality. Note that this test, when running correctly, will emit a Warning and a Hint, because the limit squashes the requested number of threads.

[jlpeyton]

LGTM

runtime/src/i18n/en_US.txt

SubmitBugReport              "Please submit a bug report with this message, compile and run "
                              "commands used, and machine configuration info including native "
                              "compiler and operating system versions. Faster response will be "
                              "obtained by including all program sources. For information on "
                              "submitting this issue, please see "
                              "https://bugs.llvm.org/."
 OBSOLETE                     "Check NLSPATH environment variable, its value is \"%1$s\"."
 ChangeStackLimit             "Please try changing the shell stack limit or adjusting the "
                              "OMP_STACKSIZE environment variable."
-Unset_ALL_THREADS            "Consider unsetting KMP_ALL_THREADS and OMP_THREAD_LIMIT (if either is set)."
+Unset_ALL_THREADS            "Consider unsetting KMP_DEVICE_THREAD_LIMIT (KMP_ALL_THREADS) and OMP_THREAD_LIMIT (if either is set)."
 Set_ALL_THREADPRIVATE        "Consider setting KMP_ALL_THREADPRIVATE to a value larger than %1$d."
 PossibleSystemLimitOnThreads "This could also be due to a system-related limit on the number of threads."
 DuplicateLibrary             "This means that multiple copies of the OpenMP runtime have been "
                              "linked into the program. That is dangerous, since it can degrade "
                              "performance or cause incorrect results. "
                              "The best thing to do is to ensure that only a single OpenMP runtime is "
                              "linked into the process, e.g. by avoiding static linking of the OpenMP "
                              "runtime in any library. As an unsafe, unsupported, undocumented workaround "

runtime/src/kmp.h

typedef struct kmp_base_root {
   int r_in_parallel; /* keeps a count of active parallel regions per root */
   // GEH: This is misnamed, should be r_active_levels
   kmp_team_t *r_root_team;
   kmp_team_t *r_hot_team;
   kmp_info_t *r_uber_thread;
   kmp_lock_t r_begin_lock;
   volatile int r_begin;
   int r_blocktime; /* blocktime for this root and descendants */
+  int r_cg_nthreads; // count of active threads in a contention group
 } kmp_base_root_t;
 
 typedef union KMP_ALIGN_CACHE kmp_root {
   kmp_base_root_t r;
   double r_align; /* use worst case alignment */
   char r_pad[KMP_PAD(kmp_base_root_t, CACHE_LINE)];
 } kmp_root_t;
 

 extern int __kmp_allThreadsSpecified;
 
 extern size_t __kmp_align_alloc;
 /* following data protected by initialization routines */
 extern int __kmp_xproc; /* number of processors in the system */
 extern int __kmp_avail_proc; /* number of processors available to the process */
 extern size_t __kmp_sys_min_stksize; /* system-defined minimum stack size */
 extern int __kmp_sys_max_nth; /* system-imposed maximum number of threads */
-extern int
-    __kmp_max_nth; /* maximum total number of concurrently-existing threads */
+// maximum total number of concurrently-existing threads on device
+extern int __kmp_max_nth;
+// maximum total number of concurrently-existing threads in a contention group
+extern int __kmp_cg_max_nth;
 extern int __kmp_threads_capacity; /* capacity of the arrays __kmp_threads and
                                       __kmp_root */
 extern int __kmp_dflt_team_nth; /* default number of threads in a parallel
                                    region a la OMP_NUM_THREADS */
 extern int __kmp_dflt_team_nth_ub; /* upper bound on "" determined at serial
                                       initialization */
 extern int __kmp_tp_capacity; /* capacity of __kmp_threads if threadprivate is
                                  used (fixed) */

runtime/src/kmp_ftn_entry.h

 int FTN_STDCALL xexpand(FTN_GET_THREAD_LIMIT)(void) {
 #ifdef KMP_STUB
   return 1; // TO DO: clarify whether it returns 1 or 0?
 #else
   if (!__kmp_init_serial) {
     __kmp_serial_initialize();
   };
   /* global ICV */
-  return __kmp_max_nth;
+  return __kmp_cg_max_nth;
 #endif
 }
 
 int FTN_STDCALL xexpand(FTN_IN_FINAL)(void) {
 #ifdef KMP_STUB
   return 0; // TO DO: clarify whether it returns 1 or 0?
 #else
   if (!TCR_4(__kmp_init_parallel)) {

runtime/src/kmp_global.cpp

 
 int __kmp_generate_warnings = kmp_warnings_low;
 int __kmp_reserve_warn = 0;
 int __kmp_xproc = 0;
 int __kmp_avail_proc = 0;
 size_t __kmp_sys_min_stksize = KMP_MIN_STKSIZE;
 int __kmp_sys_max_nth = KMP_MAX_NTH;
 int __kmp_max_nth = 0;
+int __kmp_cg_max_nth = 0;
 int __kmp_threads_capacity = 0;
 int __kmp_dflt_team_nth = 0;
 int __kmp_dflt_team_nth_ub = 0;
 int __kmp_tp_capacity = 0;
 int __kmp_tp_cached = 0;
 int __kmp_dflt_nested = FALSE;
 int __kmp_dispatch_num_buffers = KMP_DFLT_DISP_NUM_BUFF;
 int __kmp_dflt_max_active_levels =

runtime/src/kmp_runtime.cpp

if (set_nthreads > 2) {
                       "reservation to %d threads\n",
                       master_tid, new_nthreads));
       }
     }
   } else {
     KMP_ASSERT(0);
   }
 
-  // Respect KMP_ALL_THREADS, KMP_DEVICE_THREAD_LIMIT, OMP_THREAD_LIMIT.
+  // Respect KMP_ALL_THREADS/KMP_DEVICE_THREAD_LIMIT.
   if (__kmp_nth + new_nthreads -
           (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
       __kmp_max_nth) {
     int tl_nthreads = __kmp_max_nth - __kmp_nth +
                       (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
     if (tl_nthreads <= 0) {
       tl_nthreads = 1;
     }
 
     // If dyn-var is false, emit a 1-time warning.
     if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) {
       __kmp_reserve_warn = 1;
       __kmp_msg(kmp_ms_warning,
                 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads),
                 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
     }
     if (tl_nthreads == 1) {
-      KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_ALL_THREADS reduced "
-                    "reservation to 1 thread\n",
+      KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT "
+                    "reduced reservation to 1 thread\n",
                     master_tid));
       return 1;
     }
-    KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_ALL_THREADS reduced "
+    KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced "
+                  "reservation to %d threads\n",
+                  master_tid, tl_nthreads));
+    new_nthreads = tl_nthreads;
+  }
+
+  // Respect OMP_THREAD_LIMIT
+  if (root->r.r_cg_nthreads + new_nthreads -
+          (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
+      __kmp_cg_max_nth) {
+    int tl_nthreads = __kmp_cg_max_nth - root->r.r_cg_nthreads +
+                      (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
+    if (tl_nthreads <= 0) {
+      tl_nthreads = 1;
+    }
+
+    // If dyn-var is false, emit a 1-time warning.
+    if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) {
+      __kmp_reserve_warn = 1;
+      __kmp_msg(kmp_ms_warning,
+                KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads),
+                KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
+    }
+    if (tl_nthreads == 1) {
+      KC_TRACE(10, ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT "
+                    "reduced reservation to 1 thread\n",
+                    master_tid));
+      return 1;
+    }
+    KC_TRACE(10, ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced "
                   "reservation to %d threads\n",
                   master_tid, tl_nthreads));
     new_nthreads = tl_nthreads;
   }
 
   // Check if the threads array is large enough, or needs expanding.
   // See comment in __kmp_register_root() about the adjustment if
   // __kmp_threads[0] == NULL.
static void __kmp_initialize_root(kmp_root_t *root) {
 
   /* setup the root state structure */
   __kmp_init_lock(&root->r.r_begin_lock);
   root->r.r_begin = FALSE;
   root->r.r_active = FALSE;
   root->r.r_in_parallel = 0;
   root->r.r_blocktime = __kmp_dflt_blocktime;
   root->r.r_nested = __kmp_dflt_nested;
+  root->r.r_cg_nthreads = 1;
 
   /* setup the root team for this task */
   /* allocate the root team structure */
   KF_TRACE(10, ("__kmp_initialize_root: before root_team\n"));
 
   root_team =
       __kmp_allocate_team(root,
                           1, // new_nproc
while (1) {
     int nTarget;
     int minimumRequiredCapacity;
     int newCapacity;
     kmp_info_t **newThreads;
     kmp_root_t **newRoot;
 
     // Note that __kmp_threads_capacity is not bounded by __kmp_max_nth. If
     // __kmp_max_nth is set to some value less than __kmp_sys_max_nth by the
-    // user via OMP_THREAD_LIMIT, then __kmp_threads_capacity may become
+    // user via KMP_DEVICE_THREAD_LIMIT, then __kmp_threads_capacity may become
     // > __kmp_max_nth in one of two ways:
     //
     // 1) The initialization thread (gtid = 0) exits.  __kmp_threads[0]
     //    may not be resused by another thread, so we may need to increase
     //    __kmp_threads_capacity to __kmp_max_nth + 1.
     //
     // 2) New foreign root(s) are encountered.  We always register new foreign
     //    roots. This may cause a smaller # of threads to be allocated at
#if OMPT_SUPPORT
   if (ompt_enabled && ompt_callbacks.ompt_callback(ompt_event_thread_end)) {
     int gtid = __kmp_get_gtid();
     __ompt_thread_end(ompt_thread_initial, gtid);
   }
 #endif
 
   TCW_4(__kmp_nth,
         __kmp_nth - 1); // __kmp_reap_thread will decrement __kmp_all_nth.
+  root->r.r_cg_nthreads--;
+
   __kmp_reap_thread(root->r.r_uber_thread, 1);
 
   // We canot put root thread to __kmp_thread_pool, so we have to reap it istead
   // of freeing.
   root->r.r_uber_thread = NULL;
   /* mark root as no longer in use */
   root->r.r_begin = FALSE;
 
if (__kmp_thread_pool) {
     KMP_DEBUG_ASSERT(__kmp_thread_pool_nth >= 0);
 
     /* setup the thread structure */
     __kmp_initialize_info(new_thr, team, new_tid,
                           new_thr->th.th_info.ds.ds_gtid);
     KMP_DEBUG_ASSERT(new_thr->th.th_serial_team);
 
     TCW_4(__kmp_nth, __kmp_nth + 1);
+    root->r.r_cg_nthreads++;
 
     new_thr->th.th_task_state = 0;
     new_thr->th.th_task_state_top = 0;
     new_thr->th.th_task_state_stack_sz = 4;
 
 #ifdef KMP_ADJUST_BLOCKTIME
     /* Adjust blocktime back to zero if necessary */
     /* Middle initialization might not have occurred yet */
#endif
   TCW_4(new_thr->th.th_in_pool, FALSE);
   new_thr->th.th_active_in_pool = FALSE;
   TCW_4(new_thr->th.th_active, TRUE);
 
   /* adjust the global counters */
   __kmp_all_nth++;
   __kmp_nth++;
 
+  root->r.r_cg_nthreads++;
+
   // if __kmp_adjust_gtid_mode is set, then we use method #1 (sp search) for low
   // numbers of procs, and method #2 (keyed API call) for higher numbers.
   if (__kmp_adjust_gtid_mode) {
     if (__kmp_all_nth >= __kmp_tls_gtid_min) {
       if (TCR_4(__kmp_gtid_mode) != 2) {
         TCW_4(__kmp_gtid_mode, 2);
       }
     } else {

 // back on the hot team in reverse order.  This could cause bad cache
 // locality problems on programs where the size of the hot team regularly
 // grew and shrunk.
 //
 // Now, for single-level parallelism, the OMP tid is alway == gtid.
 void __kmp_free_thread(kmp_info_t *this_th) {
   int gtid;
   kmp_info_t **scan;
+  kmp_root_t *root = this_th->th.th_root;
 
   KA_TRACE(20, ("__kmp_free_thread: T#%d putting T#%d back on free pool.\n",
                 __kmp_get_gtid(), this_th->th.th_info.ds.ds_gtid));
 
   KMP_DEBUG_ASSERT(this_th);
 
   // When moving thread to pool, switch thread to wait on own b_go flag, and
   // uninitialized (NULL team).
void __kmp_free_thread(kmp_info_t *this_th) {
   __kmp_thread_pool_insert_pt = *scan = this_th;
   KMP_DEBUG_ASSERT((this_th->th.th_next_pool == NULL) ||
                    (this_th->th.th_info.ds.ds_gtid <
                     this_th->th.th_next_pool->th.th_info.ds.ds_gtid));
   TCW_4(this_th->th.th_in_pool, TRUE);
   __kmp_thread_pool_nth++;
 
   TCW_4(__kmp_nth, __kmp_nth - 1);
+  root->r.r_cg_nthreads--;
 
 #ifdef KMP_ADJUST_BLOCKTIME
   /* Adjust blocktime back to user setting or default if necessary */
   /* Middle initialization might never have occurred                */
   if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
     KMP_DEBUG_ASSERT(__kmp_avail_proc > 0);
     if (__kmp_nth <= __kmp_avail_proc) {
       __kmp_zero_bt = FALSE;
#endif
   __kmp_dflt_team_nth_ub = __kmp_xproc;
   if (__kmp_dflt_team_nth_ub < KMP_MIN_NTH) {
     __kmp_dflt_team_nth_ub = KMP_MIN_NTH;
   }
   if (__kmp_dflt_team_nth_ub > __kmp_sys_max_nth) {
     __kmp_dflt_team_nth_ub = __kmp_sys_max_nth;
   }
   __kmp_max_nth = __kmp_sys_max_nth;
+  __kmp_cg_max_nth = __kmp_sys_max_nth;
 
   // Three vars below moved here from __kmp_env_initialize() "KMP_BLOCKTIME"
   // part
   __kmp_dflt_blocktime = KMP_DEFAULT_BLOCKTIME;
 #if KMP_USE_MONITOR
   __kmp_monitor_wakeups =
       KMP_WAKEUPS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups);
   __kmp_bt_intervals =
if (num_threads == 0) {
     if (num_teams * num_threads > __kmp_max_nth) {
       // adjust num_threads w/o warning as it is not user setting
       num_threads = __kmp_max_nth / num_teams;
     }
   } else {
     if (num_teams * num_threads > __kmp_max_nth) {
       int new_threads = __kmp_max_nth / num_teams;
       if (!__kmp_reserve_warn) { // user asked for too many threads
-        __kmp_reserve_warn = 1; // that conflicts with OMP_THREAD_LIMIT
+        __kmp_reserve_warn = 1; // that conflicts with KMP_DEVICE_THREAD_LIMIT
         __kmp_msg(kmp_ms_warning,
                   KMP_MSG(CantFormThrTeam, num_threads, new_threads),
                   KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
       }
       num_threads = new_threads;
     }
   }
   thr->th.th_teams_size.nth = num_threads;

runtime/src/kmp_settings.cpp

if (__kmp_env_format) {
     return;
   }
 } // __kmp_stg_print_size
 
 // =============================================================================
 // Parse and print functions.
 
 // -----------------------------------------------------------------------------
-// KMP_ALL_THREADS, KMP_DEVICE_THREAD_LIMIT, OMP_THREAD_LIMIT
+// KMP_DEVICE_THREAD_LIMIT, KMP_ALL_THREADS
 
 static void __kmp_stg_parse_device_thread_limit(char const *name,
                                                 char const *value, void *data) {
   kmp_setting_t **rivals = (kmp_setting_t **)data;
   int rc;
   if (strcmp(name, "KMP_ALL_THREADS") == 0) {
     KMP_INFORM(EnvVarDeprecated, name, "KMP_DEVICE_THREAD_LIMIT");
   }

 } // __kmp_stg_parse_device_thread_limit
 
 static void __kmp_stg_print_device_thread_limit(kmp_str_buf_t *buffer,
                                                 char const *name, void *data) {
   __kmp_stg_print_int(buffer, name, __kmp_max_nth);
 } // __kmp_stg_print_device_thread_limit
 
 // -----------------------------------------------------------------------------
+// OMP_THREAD_LIMIT
+static void __kmp_stg_parse_thread_limit(char const *name, char const *value,
+                                         void *data) {
+  __kmp_stg_parse_int(name, value, 1, __kmp_sys_max_nth, &__kmp_cg_max_nth);
+  K_DIAG(1, ("__kmp_cg_max_nth == %d\n", __kmp_cg_max_nth));
+
+} // __kmp_stg_parse_thread_limit
+
+static void __kmp_stg_print_thread_limit(kmp_str_buf_t *buffer,
+                                         char const *name, void *data) {
+  __kmp_stg_print_int(buffer, name, __kmp_cg_max_nth);
+} // __kmp_stg_print_thread_limit
+
+// -----------------------------------------------------------------------------
 // KMP_BLOCKTIME
 
 static void __kmp_stg_parse_blocktime(char const *name, char const *value,
                                       void *data) {
   __kmp_dflt_blocktime = __kmp_convert_to_milliseconds(value);
   if (__kmp_dflt_blocktime < 0) {
     __kmp_dflt_blocktime = KMP_DEFAULT_BLOCKTIME;
     __kmp_msg(kmp_ms_warning, KMP_MSG(InvalidValue, name, value),
#if OMP_40_ENABLED
      __kmp_stg_print_default_device, NULL, 0, 0},
 #endif
 #if OMP_45_ENABLED
     {"OMP_MAX_TASK_PRIORITY", __kmp_stg_parse_max_task_priority,
      __kmp_stg_print_max_task_priority, NULL, 0, 0},
     {"KMP_TASKLOOP_MIN_TASKS", __kmp_stg_parse_taskloop_min_tasks,
      __kmp_stg_print_taskloop_min_tasks, NULL, 0, 0},
 #endif
-    {"OMP_THREAD_LIMIT", __kmp_stg_parse_device_thread_limit,
-     __kmp_stg_print_device_thread_limit, NULL, 0, 0},
+    {"OMP_THREAD_LIMIT", __kmp_stg_parse_thread_limit,
+     __kmp_stg_print_thread_limit, NULL, 0, 0},
     {"OMP_WAIT_POLICY", __kmp_stg_parse_wait_policy,
      __kmp_stg_print_wait_policy, NULL, 0, 0},
     {"KMP_DISP_NUM_BUFFERS", __kmp_stg_parse_disp_buffers,
      __kmp_stg_print_disp_buffers, NULL, 0, 0},
 #if KMP_NESTED_HOT_TEAMS
     {"KMP_HOT_TEAMS_MAX_LEVEL", __kmp_stg_parse_hot_teams_level,
      __kmp_stg_print_hot_teams_level, NULL, 0, 0},
     {"KMP_HOT_TEAMS_MODE", __kmp_stg_parse_hot_teams_mode,
#endif
       rivals[i++] = NULL;
 
       kmp_library->data = &kmp_data;
       if (omp_wait_policy != NULL) {
         omp_wait_policy->data = &omp_data;
       }; // if
     }
 
-    { // Initialize KMP_DEVICE_THREAD_LIMIT, KMP_ALL_THREADS, and
-      // OMP_THREAD_LIMIT data.
+    { // Initialize KMP_DEVICE_THREAD_LIMIT and KMP_ALL_THREADS
       kmp_setting_t *kmp_device_thread_limit =
           __kmp_stg_find("KMP_DEVICE_THREAD_LIMIT"); // 1st priority.
       kmp_setting_t *kmp_all_threads =
           __kmp_stg_find("KMP_ALL_THREADS"); // 2nd priority.
-      kmp_setting_t *omp_thread_limit =
-          __kmp_stg_find("OMP_THREAD_LIMIT"); // 3rd priority.
 
       // !!! volatile keyword is Intel (R) C Compiler bug CQ49908 workaround.
-      static kmp_setting_t *volatile rivals[4];
+      static kmp_setting_t *volatile rivals[3];
       int i = 0;
 
       rivals[i++] = kmp_device_thread_limit;
       rivals[i++] = kmp_all_threads;
-      rivals[i++] = omp_thread_limit;
       rivals[i++] = NULL;
 
       kmp_device_thread_limit->data = CCAST(kmp_setting_t **, rivals);
       kmp_all_threads->data = CCAST(kmp_setting_t **, rivals);
-      omp_thread_limit->data = CCAST(kmp_setting_t **, rivals);
     }
 
 #if KMP_AFFINITY_SUPPORTED
     { // Initialize KMP_AFFINITY, GOMP_CPU_AFFINITY, and OMP_PROC_BIND data.
       kmp_setting_t *kmp_affinity =
           __kmp_stg_find("KMP_AFFINITY"); // 1st priority.
       KMP_DEBUG_ASSERT(kmp_affinity != NULL);