[5.0] Deprecate nest-var and associated features

[tlwilmar]

Summary

Nest-var, OMP_NESTED, omp_set_nested()., and omp_get_nested() have been deprecated in the 5.0 spec. Initial nesting info is now derived from OMP_MAX_ACTIVE_LEVELS, OMP_NUM_THREADS, and OMP_PROC_BIND.

This patch deprecates the internal ICV that corresponds to nest-var, and replaces it with the max-active-levels-var ICV to determine nesting. The change still allows for use of OMP_NESTED (according to 5.0 changes), omp_get_nested, and omp_set_nested, which have had deprecation messages added to them. The change allows certain settings of OMP_NUM_THREADS, OMP_PROC_BIND, and OMP_MAX_ACTIVE_LEVELS to turn on nesting, but OMP_NESTED=0 will still force nesting to be off.

The runtime now prints informative messages about deprecation of OMP_NESTED, omp_set_nested(), and omp_get_nested(), when those environment variables or routines are used. It also prints deprecated message in output for KMP_SETTINGS and OMP_DISPLAY_ENV for OMP_NESTED. This patch also fixes OMP_DISPLAY_ENV output for OMP_TARGET_OFFLOAD.

[tlwilmar]

*ping*

[hbae]

LGTM.

runtime/src/i18n/en_US.txt

 EnvVarDeprecated             "%1$s variable deprecated, please use %2$s instead."
 RedMethodNotSupported        "KMP_FORCE_REDUCTION: %1$s method is not supported; using critical."
 AffHWSubsetNoHWLOC           "KMP_HW_SUBSET ignored: unsupported item requested for non-HWLOC topology method (KMP_TOPOLOGY_METHOD)"
 AffHWSubsetManyNodes         "KMP_HW_SUBSET ignored: too many NUMA Nodes requested."
 AffHWSubsetManyTiles         "KMP_HW_SUBSET ignored: too many L2 Caches requested."
 AffHWSubsetManyProcs         "KMP_HW_SUBSET ignored: too many Procs requested."
 HierSchedInvalid             "Hierarchy ignored: unsupported level: %1$s."
 AffFormatDefault             "OMP: pid %1$s tid %2$s thread %3$s bound to OS proc set {%4$s}"
-
+APIDeprecated                "%1$s routine deprecated, please use %2$s instead."
 
 # --------------------------------------------------------------------------------------------------
 -*- HINTS -*-
 # --------------------------------------------------------------------------------------------------
 
 # Hints. Hint may be printed after a message. Usually it is longer explanation text or suggestion.
 # To maintain hint numbers (they are visible to customers), add new hints to the end.
 
BadExeFormat                 "System error #193 is \"Bad format of EXE or DLL file\". "
                              "Check whether \"%1$s\" is a file for %2$s architecture."
 SystemLimitOnThreads         "System-related limit on the number of threads."
 
 
 
 # --------------------------------------------------------------------------------------------------
 # end of file #
 # --------------------------------------------------------------------------------------------------
-

runtime/src/kmp.h

 } kmp_bar_pat_e;
 
 #define KMP_BARRIER_ICV_PUSH 1
 
 /* Record for holding the values of the internal controls stack records */
 typedef struct kmp_internal_control {
   int serial_nesting_level; /* corresponds to the value of the
                                th_team_serialized field */
-  kmp_int8 nested; /* internal control for nested parallelism (per thread) */
   kmp_int8 dynamic; /* internal control for dynamic adjustment of threads (per
                        thread) */
   kmp_int8
       bt_set; /* internal control for whether blocktime is explicitly set */
   int blocktime; /* internal control for blocktime */
 #if KMP_USE_MONITOR
   int bt_intervals; /* internal control for blocktime intervals */
 #endif
#define get__blocktime(xteam, xtid)                                            \
   ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.blocktime)
 #define get__bt_set(xteam, xtid)                                               \
   ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.bt_set)
 #if KMP_USE_MONITOR
 #define get__bt_intervals(xteam, xtid)                                         \
   ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.bt_intervals)
 #endif
 
-#define get__nested_2(xteam, xtid)                                             \
-  ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.nested)
 #define get__dynamic_2(xteam, xtid)                                            \
   ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.dynamic)
 #define get__nproc_2(xteam, xtid)                                              \
   ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.nproc)
 #define get__sched_2(xteam, xtid)                                              \
   ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.sched)
 
 #define set__blocktime_team(xteam, xtid, xval)                                 \
   (((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.blocktime) =     \
        (xval))
 
 #if KMP_USE_MONITOR
 #define set__bt_intervals_team(xteam, xtid, xval)                              \
   (((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.bt_intervals) =  \
        (xval))
 #endif
 
 #define set__bt_set_team(xteam, xtid, xval)                                    \
   (((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.bt_set) = (xval))
 
-#define set__nested(xthread, xval)                                             \
-  (((xthread)->th.th_current_task->td_icvs.nested) = (xval))
-#define get__nested(xthread)                                                   \
-  (((xthread)->th.th_current_task->td_icvs.nested) ? (FTN_TRUE) : (FTN_FALSE))
-
 #define set__dynamic(xthread, xval)                                            \
   (((xthread)->th.th_current_task->td_icvs.dynamic) = (xval))
 #define get__dynamic(xthread)                                                  \
   (((xthread)->th.th_current_task->td_icvs.dynamic) ? (FTN_TRUE) : (FTN_FALSE))
 
 #define set__nproc(xthread, xval)                                              \
   (((xthread)->th.th_current_task->td_icvs.nproc) = (xval))
 
 #define set__thread_limit(xthread, xval)                                       \
   (((xthread)->th.th_current_task->td_icvs.thread_limit) = (xval))
 
 #define set__max_active_levels(xthread, xval)                                  \
   (((xthread)->th.th_current_task->td_icvs.max_active_levels) = (xval))
 
+#define get__max_active_levels(xthread)                                        \
+  ((xthread)->th.th_current_task->td_icvs.max_active_levels)
+
 #define set__sched(xthread, xval)                                              \
   (((xthread)->th.th_current_task->td_icvs.sched) = (xval))
 
 #if OMP_40_ENABLED
 
 #define set__proc_bind(xthread, xval)                                          \
   (((xthread)->th.th_current_task->td_icvs.proc_bind) = (xval))
 #define get__proc_bind(xthread)                                                \

 } kmp_global_t;
 
 typedef struct kmp_base_root {
   // TODO: GEH - combine r_active with r_in_parallel then r_active ==
   // (r_in_parallel>= 0)
   // TODO: GEH - then replace r_active with t_active_levels if we can to reduce
   // the synch overhead or keeping r_active
   volatile int r_active; /* TRUE if some region in a nest has > 1 thread */
-  // GEH: This is misnamed, should be r_in_parallel
-  volatile int r_nested; // TODO: GEH - This is unused, just remove it entirely.
   // keeps a count of active parallel regions per root
   std::atomic<int> r_in_parallel;
   // GEH: This is misnamed, should be r_active_levels
   kmp_team_t *r_root_team;
   kmp_team_t *r_hot_team;
   kmp_info_t *r_uber_thread;
   kmp_lock_t r_begin_lock;
   volatile int r_begin;

 extern int __kmp_dflt_team_nth; /* default number of threads in a parallel
                                    region a la OMP_NUM_THREADS */
 extern int __kmp_dflt_team_nth_ub; /* upper bound on "" determined at serial
                                       initialization */
 extern int __kmp_tp_capacity; /* capacity of __kmp_threads if threadprivate is
                                  used (fixed) */
 extern int __kmp_tp_cached; /* whether threadprivate cache has been created
                                (__kmpc_threadprivate_cached()) */
-extern int __kmp_dflt_nested; /* nested parallelism enabled by default a la
-                                 OMP_NESTED */
 extern int __kmp_dflt_blocktime; /* number of milliseconds to wait before
                                     blocking (env setting) */
 #if KMP_USE_MONITOR
 extern int
     __kmp_monitor_wakeups; /* number of times monitor wakes up per second */
 extern int __kmp_bt_intervals; /* number of monitor timestamp intervals before
                                   blocking */
 #endif

 extern int __kmp_tls_gtid_min; /* #threads below which use sp search for gtid */
 extern int __kmp_foreign_tp; // If true, separate TP var for each foreign thread
 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
 extern int __kmp_inherit_fp_control; // copy fp creg(s) parent->workers at fork
 extern kmp_int16 __kmp_init_x87_fpu_control_word; // init thread's FP ctrl reg
 extern kmp_uint32 __kmp_init_mxcsr; /* init thread's mxscr */
 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
 
-extern int __kmp_dflt_max_active_levels; /* max_active_levels for nested
-                                            parallelism enabled by default via
-                                            OMP_MAX_ACTIVE_LEVELS */
+// max_active_levels for nested parallelism enabled by default via
+// OMP_MAX_ACTIVE_LEVELS, OMP_NESTED, OMP_NUM_THREADS, and OMP_PROC_BIND
+extern int __kmp_dflt_max_active_levels;
+// Indicates whether value of __kmp_dflt_max_active_levels was already
+// explicitly set by OMP_MAX_ACTIVE_LEVELS or OMP_NESTED=false
+extern bool __kmp_dflt_max_active_levels_set;
 extern int __kmp_dispatch_num_buffers; /* max possible dynamic loops in
                                           concurrent execution per team */
 #if KMP_NESTED_HOT_TEAMS
 extern int __kmp_hot_teams_mode;
 extern int __kmp_hot_teams_max_level;
 #endif
 
 #if KMP_OS_LINUX

runtime/src/kmp_csupport.cpp

 void ompc_set_nested(int flag) {
   kmp_info_t *thread;
 
   /* For the thread-private internal controls implementation */
   thread = __kmp_entry_thread();
 
   __kmp_save_internal_controls(thread);
 
-  set__nested(thread, flag ? TRUE : FALSE);
+  set__max_active_levels(thread, flag ? __kmp_dflt_max_active_levels : 1);
 }
 
 void ompc_set_max_active_levels(int max_active_levels) {
   /* TO DO */
   /* we want per-task implementation of this internal control */
 
   /* For the per-thread internal controls implementation */
   __kmp_set_max_active_levels(__kmp_entry_gtid(), max_active_levels);

runtime/src/kmp_ftn_entry.h

#else
   if (!TCR_4(__kmp_init_middle)) {
     __kmp_middle_initialize();
   }
   return __kmp_avail_proc;
 #endif
 }
 
 void FTN_STDCALL KMP_EXPAND_NAME(FTN_SET_NESTED)(int KMP_DEREF flag) {
+  KMP_INFORM(APIDeprecated, "omp_set_nested", "omp_set_max_active_levels");
 #ifdef KMP_STUB
   __kmps_set_nested(KMP_DEREF flag);
 #else
   kmp_info_t *thread;
   /* For the thread-private internal controls implementation */
   thread = __kmp_entry_thread();
   __kmp_save_internal_controls(thread);
-  set__nested(thread, ((KMP_DEREF flag) ? TRUE : FALSE));
+  // Somewhat arbitrarily decide where to get a value for max_active_levels
+  int max_active_levels = get__max_active_levels(thread);
+  if (max_active_levels == 1)
+    max_active_levels = KMP_MAX_ACTIVE_LEVELS_LIMIT;
+  set__max_active_levels(thread, (KMP_DEREF flag) ? max_active_levels : 1);
 #endif
 }
 
 int FTN_STDCALL KMP_EXPAND_NAME(FTN_GET_NESTED)(void) {
+  KMP_INFORM(APIDeprecated, "omp_get_nested", "omp_get_max_active_levels");
 #ifdef KMP_STUB
   return __kmps_get_nested();
 #else
   kmp_info_t *thread;
   thread = __kmp_entry_thread();
-  return get__nested(thread);
+  return get__max_active_levels(thread) > 1;
 #endif
 }
 
 void FTN_STDCALL KMP_EXPAND_NAME(FTN_SET_DYNAMIC)(int KMP_DEREF flag) {
 #ifdef KMP_STUB
   __kmps_set_dynamic(KMP_DEREF flag ? TRUE : FALSE);
 #else
   kmp_info_t *thread;

runtime/src/kmp_global.cpp

 int __kmp_max_nth = 0;
 int __kmp_cg_max_nth = 0;
 int __kmp_teams_max_nth = 0;
 int __kmp_threads_capacity = 0;
 int __kmp_dflt_team_nth = 0;
 int __kmp_dflt_team_nth_ub = 0;
 int __kmp_tp_capacity = 0;
 int __kmp_tp_cached = 0;
-int __kmp_dflt_nested = FALSE;
 int __kmp_dispatch_num_buffers = KMP_DFLT_DISP_NUM_BUFF;
-int __kmp_dflt_max_active_levels =
-    KMP_MAX_ACTIVE_LEVELS_LIMIT; /* max_active_levels limit */
+int __kmp_dflt_max_active_levels = 1; // Nesting off by default
+bool __kmp_dflt_max_active_levels_set = false; // Don't override set value
 #if KMP_NESTED_HOT_TEAMS
 int __kmp_hot_teams_mode = 0; /* 0 - free extra threads when reduced */
 /* 1 - keep extra threads when reduced */
 int __kmp_hot_teams_max_level = 1; /* nesting level of hot teams */
 #endif
 enum library_type __kmp_library = library_none;
 enum sched_type __kmp_sched =
     kmp_sch_default; /* scheduling method for runtime scheduling */

runtime/src/kmp_runtime.cpp

#endif
               : get__nproc_2(
                     parent_team,
                     master_tid); // TODO: get nproc directly from current task
 
       // Check if we need to take forkjoin lock? (no need for serialized
       // parallel out of teams construct). This code moved here from
       // __kmp_reserve_threads() to speedup nested serialized parallels.
       if (nthreads > 1) {
-        if ((!get__nested(master_th) && (root->r.r_in_parallel
+        if ((get__max_active_levels(master_th) == 1 && (root->r.r_in_parallel
 #if OMP_40_ENABLED
                                                         && !enter_teams
 #endif /* OMP_40_ENABLED */
                                                         )) ||
             (__kmp_library == library_serial)) {
           KC_TRACE(10, ("__kmp_fork_call: T#%d serializing team; requested %d"
                         " threads\n",
                         gtid, nthreads));
           nthreads = 1;
         }
       }
       if (nthreads > 1) {
kmp_r_sched_t r_sched =
       __kmp_get_schedule_global(); // get current state of scheduling globals
 
 #if OMP_40_ENABLED
   KMP_DEBUG_ASSERT(__kmp_nested_proc_bind.used > 0);
 #endif /* OMP_40_ENABLED */
 
   kmp_internal_control_t g_icvs = {
     0, // int serial_nesting_level; //corresponds to value of th_team_serialized
-    (kmp_int8)__kmp_dflt_nested, // int nested; //internal control
-    // for nested parallelism (per thread)
     (kmp_int8)__kmp_global.g.g_dynamic, // internal control for dynamic
     // adjustment of threads (per thread)
     (kmp_int8)__kmp_env_blocktime, // int bt_set; //internal control for
     // whether blocktime is explicitly set
     __kmp_dflt_blocktime, // int blocktime; //internal control for blocktime
 #if KMP_USE_MONITOR
     __kmp_bt_intervals, // int bt_intervals; //internal control for blocktime
 // intervals
static void __kmp_initialize_root(kmp_root_t *root) {
   KMP_ASSERT(!root->r.r_begin);
 
   /* setup the root state structure */
   __kmp_init_lock(&root->r.r_begin_lock);
   root->r.r_begin = FALSE;
   root->r.r_active = FALSE;
   root->r.r_in_parallel = 0;
   root->r.r_blocktime = __kmp_dflt_blocktime;
-  root->r.r_nested = __kmp_dflt_nested;
 
   /* setup the root team for this task */
   /* allocate the root team structure */
   KF_TRACE(10, ("__kmp_initialize_root: before root_team\n"));
 
   root_team =
       __kmp_allocate_team(root,
                           1, // new_nproc
for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
       kmp_root_t const *root = __kmp_root[gtid];
       if (root != NULL) {
         __kmp_printf("GTID %2d %p:\n", gtid, root);
         __kmp_print_structure_team("    Root Team:    ", root->r.r_root_team);
         __kmp_print_structure_team("    Hot Team:     ", root->r.r_hot_team);
         __kmp_print_structure_thread("    Uber Thread:  ",
                                      root->r.r_uber_thread);
         __kmp_printf("    Active?:      %2d\n", root->r.r_active);
-        __kmp_printf("    Nested?:      %2d\n", root->r.r_nested);
         __kmp_printf("    In Parallel:  %2d\n",
                      KMP_ATOMIC_LD_RLX(&root->r.r_in_parallel));
         __kmp_printf("\n");
         __kmp_print_structure_team_accum(list, root->r.r_root_team);
         __kmp_print_structure_team_accum(list, root->r.r_hot_team);
       }
     }
   } else {

runtime/src/kmp_settings.cpp

static void __kmp_stg_print_warnings(kmp_str_buf_t *buffer, char const *name,
   __kmp_stg_print_bool(buffer, name, __kmp_generate_warnings);
 } // __kmp_stg_print_warnings
 
 // -----------------------------------------------------------------------------
 // OMP_NESTED, OMP_NUM_THREADS
 
 static void __kmp_stg_parse_nested(char const *name, char const *value,
                                    void *data) {
-  __kmp_stg_parse_bool(name, value, &__kmp_dflt_nested);
+  int nested;
+  KMP_INFORM(EnvVarDeprecated, name, "OMP_MAX_ACTIVE_LEVELS");
+  __kmp_stg_parse_bool(name, value, &nested);
+  if (nested) {
+    if (!__kmp_dflt_max_active_levels_set)
+      __kmp_dflt_max_active_levels = KMP_MAX_ACTIVE_LEVELS_LIMIT;
+  } else { // nesting explicitly turned off
+    __kmp_dflt_max_active_levels = 1;
+    __kmp_dflt_max_active_levels_set = true;
+  }
 } // __kmp_stg_parse_nested
 
 static void __kmp_stg_print_nested(kmp_str_buf_t *buffer, char const *name,
                                    void *data) {
-  __kmp_stg_print_bool(buffer, name, __kmp_dflt_nested);
+  if (__kmp_env_format) {
+    KMP_STR_BUF_PRINT_NAME;
+  } else {
+    __kmp_str_buf_print(buffer, "   %s", name);
+  }
+  __kmp_str_buf_print(buffer, ": deprecated; max-active-levels-var=%d\n",
+                      __kmp_dflt_max_active_levels);
 } // __kmp_stg_print_nested
 
 static void __kmp_parse_nested_num_threads(const char *var, const char *env,
                                            kmp_nested_nthreads_t *nth_array) {
   const char *next = env;
   const char *scan = next;
 
   int total = 0; // Count elements that were set. It'll be used as an array size
if (*next >= '0' && *next <= '9') {
       const char *tmp = next;
       SKIP_WS(tmp);
       if ((*next == ' ' || *next == '\t') && (*tmp >= '0' && *tmp <= '9')) {
         KMP_WARNING(NthSpacesNotAllowed, var, env);
         return;
       }
     }
   }
+  if (!__kmp_dflt_max_active_levels_set && total > 1)
+    __kmp_dflt_max_active_levels = KMP_MAX_ACTIVE_LEVELS_LIMIT;
   KMP_DEBUG_ASSERT(total > 0);
   if (total <= 0) {
     KMP_WARNING(NthSyntaxError, var, env);
     return;
   }
 
   // Check if the nested nthreads array exists
   if (!nth_array->nth) {

 
 static void __kmp_stg_print_task_stealing(kmp_str_buf_t *buffer,
                                           char const *name, void *data) {
   __kmp_stg_print_int(buffer, name, __kmp_task_stealing_constraint);
 } // __kmp_stg_print_task_stealing
 
 static void __kmp_stg_parse_max_active_levels(char const *name,
                                               char const *value, void *data) {
-  __kmp_stg_parse_int(name, value, 0, KMP_MAX_ACTIVE_LEVELS_LIMIT,
-                      &__kmp_dflt_max_active_levels);
+  kmp_uint64 tmp_dflt = 0;
+  char const *msg = NULL;
+  if (!__kmp_dflt_max_active_levels_set) {
+    // Don't overwrite __kmp_dflt_max_active_levels if we get an invalid setting
+    __kmp_str_to_uint(value, &tmp_dflt, &msg);
+    if (msg != NULL) { // invalid setting; print warning and ignore
+      KMP_WARNING(ParseSizeIntWarn, name, value, msg);
+    } else if (tmp_dflt > KMP_MAX_ACTIVE_LEVELS_LIMIT) {
+      // invalid setting; print warning and ignore
+      msg = KMP_I18N_STR(ValueTooLarge);
+      KMP_WARNING(ParseSizeIntWarn, name, value, msg);
+    } else { // valid setting
+      __kmp_dflt_max_active_levels = tmp_dflt;
+      __kmp_dflt_max_active_levels_set = true;
+    }
+  }
 } // __kmp_stg_parse_max_active_levels
 
 static void __kmp_stg_print_max_active_levels(kmp_str_buf_t *buffer,
                                               char const *name, void *data) {
   __kmp_stg_print_int(buffer, name, __kmp_dflt_max_active_levels);
 } // __kmp_stg_print_max_active_levels
 
 #if OMP_40_ENABLED
static void __kmp_stg_print_target_offload(kmp_str_buf_t *buffer,
                                            char const *name, void *data) {
   const char *value = NULL;
   if (__kmp_target_offload == tgt_default)
     value = "DEFAULT";
   else if (__kmp_target_offload == tgt_mandatory)
     value = "MANDATORY";
   else if (__kmp_target_offload == tgt_disabled)
     value = "DISABLED";
-  if (value) {
-    __kmp_str_buf_print(buffer, "   %s=%s\n", name, value);
+  KMP_DEBUG_ASSERT(value);
+  if (__kmp_env_format) {
+    KMP_STR_BUF_PRINT_NAME;
+  } else {
+    __kmp_str_buf_print(buffer, "   %s", name);
   }
+  __kmp_str_buf_print(buffer, "=%s\n", value);
 } // __kmp_stg_print_target_offload
 #endif
 
 #if OMP_45_ENABLED
 // -----------------------------------------------------------------------------
 // OpenMP 4.5: OMP_MAX_TASK_PRIORITY
 static void __kmp_stg_parse_max_task_priority(char const *name,
                                               char const *value, void *data) {
if (__kmp_nested_proc_bind.size < nelem) {
               sizeof(kmp_proc_bind_t) * nelem);
       if (__kmp_nested_proc_bind.bind_types == NULL) {
         KMP_FATAL(MemoryAllocFailed);
       }
       __kmp_nested_proc_bind.size = nelem;
     }
     __kmp_nested_proc_bind.used = nelem;
 
+    if (nelem > 1 && !__kmp_dflt_max_active_levels_set)
+      __kmp_dflt_max_active_levels = KMP_MAX_ACTIVE_LEVELS_LIMIT;
+
     // Save values in the nested proc_bind array
     int i = 0;
     for (;;) {
       enum kmp_proc_bind_t bind;
 
       if ((num == (int)proc_bind_master) ||
           __kmp_match_str("master", buf, &next)) {
         buf = next;
static void __kmp_aux_env_initialize(kmp_env_blk_t *block) {
   value = __kmp_env_blk_var(block, "KMP_BLOCKTIME");
   if (value) {
     kmpc_set_blocktime(__kmp_dflt_blocktime);
   }
 
   /* OMP_NESTED */
   value = __kmp_env_blk_var(block, "OMP_NESTED");
   if (value) {
-    ompc_set_nested(__kmp_dflt_nested);
+    ompc_set_nested(__kmp_dflt_max_active_levels > 1);
   }
 
   /* OMP_DYNAMIC */
   value = __kmp_env_blk_var(block, "OMP_DYNAMIC");
   if (value) {
     ompc_set_dynamic(__kmp_global.g.g_dynamic);
   }
 }