[OpenMP][libomp] Allow reset affinity mask after parallel

[jlpeyton]

Summary

Added control to reset affinity of primary thread after outermost parallel
region to initial affinity encountered before OpenMP runtime was initialized.
KMP_AFFINITY environment variable reset/noreset modifier introduced.
Default behavior is unchanged.

[tlwilmar]

LGTM

openmp/docs/design/Runtimes.rst

 
 **Syntax:** ``KMP_AFFINITY=[<modifier>,...]<type>[,<permute>][,<offset>]``
 
 ``modifiers`` are optional strings consisting of a keyword and possibly a specifier
 
 * ``respect`` (default) and ``norespect`` - determine whether to respect the original process affinity mask.
 * ``verbose`` and ``noverbose`` (default) - determine whether to display affinity information.
 * ``warnings`` (default) and ``nowarnings`` - determine whether to display warnings during affinity detection.
+* ``reset`` and ``noreset`` (default) - determine whether to reset primary thread's affinity after outermost parallel region(s)
 * ``granularity=<specifier>`` - takes the following specifiers ``thread``, ``core`` (default), ``tile``,
   ``socket``, ``die``, ``group`` (Windows only).
   The granularity describes the lowest topology levels that OpenMP threads are allowed to float within a topology map.
   For example, if ``granularity=core``, then the OpenMP threads will be allowed to move between logical processors within
   a single core. If ``granularity=thread``, then the OpenMP threads will be restricted to a single logical processor.
 * ``proclist=[<proc_list>]`` - The ``proc_list`` is specified by
 
 +--------------------+----------------------------------------+

openmp/runtime/src/kmp.h

 extern int __kmp_affinity_warnings; /* KMP_AFFINITY warnings enabled ? */
 extern int __kmp_affinity_respect_mask; // Respect process' init affinity mask?
 extern char *__kmp_affinity_proclist; /* proc ID list */
 extern kmp_affin_mask_t *__kmp_affinity_masks;
 extern unsigned __kmp_affinity_num_masks;
 extern void __kmp_affinity_bind_thread(int which);
 
 extern kmp_affin_mask_t *__kmp_affin_fullMask;
+extern kmp_affin_mask_t *__kmp_affin_origMask;
 extern char *__kmp_cpuinfo_file;
+extern bool __kmp_affin_reset;
 
 #endif /* KMP_AFFINITY_SUPPORTED */
 
 // This needs to be kept in sync with the values in omp.h !!!
 typedef enum kmp_proc_bind_t {
   proc_bind_false = 0,
   proc_bind_true,
   proc_bind_primary,

 static inline void __kmp_assign_root_init_mask() {
   int gtid = __kmp_entry_gtid();
   kmp_root_t *r = __kmp_threads[gtid]->th.th_root;
   if (r->r.r_uber_thread == __kmp_threads[gtid] && !r->r.r_affinity_assigned) {
     __kmp_affinity_set_init_mask(gtid, TRUE);
     r->r.r_affinity_assigned = TRUE;
   }
 }
+static inline void __kmp_reset_root_init_mask(int gtid) {
+  kmp_info_t *th = __kmp_threads[gtid];
+  kmp_root_t *r = th->th.th_root;
+  if (r->r.r_uber_thread == th && r->r.r_affinity_assigned) {
+    __kmp_set_system_affinity(__kmp_affin_origMask, FALSE);
+    KMP_CPU_COPY(th->th.th_affin_mask, __kmp_affin_origMask);
+    r->r.r_affinity_assigned = FALSE;
+  }
+}
 #else /* KMP_AFFINITY_SUPPORTED */
 #define __kmp_assign_root_init_mask() /* Nothing */
+static inline void __kmp_reset_root_init_mask(int gtid) {}
 #endif /* KMP_AFFINITY_SUPPORTED */
 // No need for KMP_AFFINITY_SUPPORTED guard as only one field in the
 // format string is for affinity, so platforms that do not support
 // affinity can still use the other fields, e.g., %n for num_threads
 extern size_t __kmp_aux_capture_affinity(int gtid, const char *format,
                                          kmp_str_buf_t *buffer);
 extern void __kmp_aux_display_affinity(int gtid, const char *format);
 

openmp/runtime/src/kmp_affinity.cpp

#endif /* KMP_GROUP_AFFINITY */
 
   return avail_proc;
 }
 
 // All of the __kmp_affinity_create_*_map() routines should allocate the
 // internal topology object and set the layer ids for it.  Each routine
 // returns a boolean on whether it was successful at doing so.
 kmp_affin_mask_t *__kmp_affin_fullMask = NULL;
+// Original mask is a subset of full mask in multiple processor groups topology
+kmp_affin_mask_t *__kmp_affin_origMask = NULL;
 
 #if KMP_USE_HWLOC
 static inline bool __kmp_hwloc_is_cache_type(hwloc_obj_t obj) {
 #if HWLOC_API_VERSION >= 0x00020000
   return hwloc_obj_type_is_cache(obj->type);
 #else
   return obj->type == HWLOC_OBJ_CACHE;
 #endif
static void __kmp_aux_affinity_initialize(void) {
 
   // Create the "full" mask - this defines all of the processors that we
   // consider to be in the machine model. If respect is set, then it is the
   // initialization thread's affinity mask. Otherwise, it is all processors that
   // we know about on the machine.
   if (__kmp_affin_fullMask == NULL) {
     KMP_CPU_ALLOC(__kmp_affin_fullMask);
   }
+  if (__kmp_affin_origMask == NULL) {
+    KMP_CPU_ALLOC(__kmp_affin_origMask);
+  }
   if (KMP_AFFINITY_CAPABLE()) {
     __kmp_get_system_affinity(__kmp_affin_fullMask, TRUE);
+    // Make a copy before possible expanding to the entire machine mask
+    __kmp_affin_origMask->copy(__kmp_affin_fullMask);
     if (__kmp_affinity_respect_mask) {
       // Count the number of available processors.
       unsigned i;
       __kmp_avail_proc = 0;
       KMP_CPU_SET_ITERATE(i, __kmp_affin_fullMask) {
         if (!KMP_CPU_ISSET(i, __kmp_affin_fullMask)) {
           continue;
         }
if (__kmp_affinity_respect_mask) {
         char buf[KMP_AFFIN_MASK_PRINT_LEN];
         __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
                                   __kmp_affin_fullMask);
         KMP_INFORM(InitOSProcSetNotRespect, "KMP_AFFINITY", buf);
       }
       __kmp_avail_proc =
           __kmp_affinity_entire_machine_mask(__kmp_affin_fullMask);
 #if KMP_OS_WINDOWS
+      if (__kmp_num_proc_groups <= 1) {
+        // Copy expanded full mask if topology has single processor group
+        __kmp_affin_origMask->copy(__kmp_affin_fullMask);
+      }
       // Set the process affinity mask since threads' affinity
       // masks must be subset of process mask in Windows* OS
       __kmp_affin_fullMask->set_process_affinity(true);
 #endif
     }
   }
 
   kmp_i18n_id_t msg_id = kmp_i18n_null;
#endif
   }
 
   // Canonicalize, print (if requested), apply KMP_HW_SUBSET, and
   // initialize other data structures which depend on the topology
   __kmp_topology->canonicalize();
   if (__kmp_affinity_verbose)
     __kmp_topology->print("KMP_AFFINITY");
   bool filtered = __kmp_topology->filter_hw_subset();
+  if (filtered) {
+#if KMP_OS_WINDOWS
+    // Copy filtered full mask if topology has single processor group
+    if (__kmp_num_proc_groups <= 1)
+#endif
+      __kmp_affin_origMask->copy(__kmp_affin_fullMask);
+  }
   if (filtered && __kmp_affinity_verbose)
     __kmp_topology->print("KMP_HW_SUBSET");
   machine_hierarchy.init(__kmp_topology->get_num_hw_threads());
   KMP_ASSERT(__kmp_avail_proc == __kmp_topology->get_num_hw_threads());
   // If KMP_AFFINITY=none, then only create the single "none" place
   // which is the process's initial affinity mask or the number of
   // hardware threads depending on respect,norespect
   if (__kmp_affinity_type == affinity_none) {
void __kmp_affinity_uninitialize(void) {
   if (__kmp_affinity_masks != NULL) {
     KMP_CPU_FREE_ARRAY(__kmp_affinity_masks, __kmp_affinity_num_masks);
     __kmp_affinity_masks = NULL;
   }
   if (__kmp_affin_fullMask != NULL) {
     KMP_CPU_FREE(__kmp_affin_fullMask);
     __kmp_affin_fullMask = NULL;
   }
+  if (__kmp_affin_origMask != NULL) {
+    KMP_CPU_FREE(__kmp_affin_origMask);
+    __kmp_affin_origMask = NULL;
+  }
   __kmp_affinity_num_masks = 0;
   __kmp_affinity_type = affinity_default;
   __kmp_affinity_num_places = 0;
   if (__kmp_affinity_proclist != NULL) {
     __kmp_free(__kmp_affinity_proclist);
     __kmp_affinity_proclist = NULL;
   }
   if (procarr != NULL) {

openmp/runtime/src/kmp_csupport.cpp

if (__kmp_tasking_mode != tskm_immediate_exec) {
       // Copy the task team from the new child / old parent team to the thread.
       this_thr->th.th_task_team =
           this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state];
       KA_TRACE(20,
                ("__kmpc_end_serialized_parallel: T#%d restoring task_team %p / "
                 "team %p\n",
                 global_tid, this_thr->th.th_task_team, this_thr->th.th_team));
     }
+#if KMP_AFFINITY_SUPPORTED
+    if (this_thr->th.th_team->t.t_level == 0 && __kmp_affin_reset) {
+      __kmp_reset_root_init_mask(global_tid);
+    }
+#endif
   } else {
     if (__kmp_tasking_mode != tskm_immediate_exec) {
       KA_TRACE(20, ("__kmpc_end_serialized_parallel: T#%d decreasing nesting "
                     "depth of serial team %p to %d\n",
                     global_tid, serial_team, serial_team->t.t_serialized));
     }
   }
 

 
 void KMP_EXPAND_NAME(ompc_display_affinity)(char const *format) {
   int gtid;
   if (!TCR_4(__kmp_init_middle)) {
     __kmp_middle_initialize();
   }
   __kmp_assign_root_init_mask();
   gtid = __kmp_get_gtid();
+#if KMP_AFFINITY_SUPPORTED
+  if (__kmp_threads[gtid]->th.th_team->t.t_level == 0 && __kmp_affin_reset) {
+    __kmp_reset_root_init_mask(gtid);
+  }
+#endif
   __kmp_aux_display_affinity(gtid, format);
 }
 
 size_t KMP_EXPAND_NAME(ompc_capture_affinity)(char *buffer, size_t buf_size,
                                               char const *format) {
   int gtid;
   size_t num_required;
   kmp_str_buf_t capture_buf;
   if (!TCR_4(__kmp_init_middle)) {
     __kmp_middle_initialize();
   }
   __kmp_assign_root_init_mask();
   gtid = __kmp_get_gtid();
+#if KMP_AFFINITY_SUPPORTED
+  if (__kmp_threads[gtid]->th.th_team->t.t_level == 0 && __kmp_affin_reset) {
+    __kmp_reset_root_init_mask(gtid);
+  }
+#endif
   __kmp_str_buf_init(&capture_buf);
   num_required = __kmp_aux_capture_affinity(gtid, format, &capture_buf);
   if (buffer && buf_size) {
     __kmp_strncpy_truncate(buffer, buf_size, capture_buf.str,
                            capture_buf.used + 1);
   }
   __kmp_str_buf_free(&capture_buf);
   return num_required;

openmp/runtime/src/kmp_ftn_entry.h

 int FTN_STDCALL FTN_GET_AFFINITY(void **mask) {
 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
   return -1;
 #else
   if (!TCR_4(__kmp_init_middle)) {
     __kmp_middle_initialize();
   }
   __kmp_assign_root_init_mask();
+  int gtid = __kmp_get_gtid();
+  if (__kmp_threads[gtid]->th.th_team->t.t_level == 0 && __kmp_affin_reset) {
+    __kmp_reset_root_init_mask(gtid);
+  }
   return __kmp_aux_get_affinity(mask);
 #endif
 }
 
 int FTN_STDCALL FTN_GET_AFFINITY_MAX_PROC(void) {
 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
   return 0;
 #else

 #ifdef KMP_STUB
   return 1;
 #else
   int gtid;
   kmp_info_t *thread;
   if (!TCR_4(__kmp_init_middle)) {
     __kmp_middle_initialize();
   }
-  __kmp_assign_root_init_mask();
   gtid = __kmp_entry_gtid();
   thread = __kmp_threads[gtid];
+#if KMP_AFFINITY_SUPPORTED
+  if (thread->th.th_team->t.t_level == 0 && !__kmp_affin_reset) {
+    __kmp_assign_root_init_mask();
+  }
+#endif
   // return thread -> th.th_team -> t.t_current_task[
   // thread->th.th_info.ds.ds_tid ] -> icvs.nproc;
   return thread->th.th_current_task->td_icvs.nproc;
 #endif
 }
 
 int FTN_STDCALL FTN_CONTROL_TOOL(int command, int modifier, void *arg) {
 #if defined(KMP_STUB) || !OMPT_SUPPORT
#ifdef KMP_STUB
   return;
 #else
   int gtid;
   if (!TCR_4(__kmp_init_middle)) {
     __kmp_middle_initialize();
   }
   __kmp_assign_root_init_mask();
   gtid = __kmp_get_gtid();
+#if KMP_AFFINITY_SUPPORTED
+  if (__kmp_threads[gtid]->th.th_team->t.t_level == 0 && __kmp_affin_reset) {
+    __kmp_reset_root_init_mask(gtid);
+  }
+#endif
   ConvertedString cformat(format, size);
   __kmp_aux_display_affinity(gtid, cformat.get());
 #endif
 }
 
 /*
  * Returns the number of characters required to hold the entire affinity format
  * specification (not including null byte) and prints the thread affinity
#else
   int gtid;
   size_t num_required;
   kmp_str_buf_t capture_buf;
   if (!TCR_4(__kmp_init_middle)) {
     __kmp_middle_initialize();
   }
   __kmp_assign_root_init_mask();
   gtid = __kmp_get_gtid();
+#if KMP_AFFINITY_SUPPORTED
+  if (__kmp_threads[gtid]->th.th_team->t.t_level == 0 && __kmp_affin_reset) {
+    __kmp_reset_root_init_mask(gtid);
+  }
+#endif
   __kmp_str_buf_init(&capture_buf);
   ConvertedString cformat(format, for_size);
   num_required = __kmp_aux_capture_affinity(gtid, cformat.get(), &capture_buf);
   if (buffer && buf_size) {
     __kmp_fortran_strncpy_truncate(buffer, buf_size, capture_buf.str,
                                    capture_buf.used);
   }
   __kmp_str_buf_free(&capture_buf);

 
 int FTN_STDCALL KMP_EXPAND_NAME(FTN_GET_NUM_PROCS)(void) {
 #ifdef KMP_STUB
   return 1;
 #else
   if (!TCR_4(__kmp_init_middle)) {
     __kmp_middle_initialize();
   }
+#if KMP_AFFINITY_SUPPORTED
+  if (!__kmp_affin_reset) {
+    // only bind root here if its affinity reset is not requested
+    int gtid = __kmp_entry_gtid();
+    kmp_info_t *thread = __kmp_threads[gtid];
+    if (thread->th.th_team->t.t_level == 0) {
       __kmp_assign_root_init_mask();
+    }
+  }
+#endif
   return __kmp_avail_proc;
 #endif
 }
 
 void FTN_STDCALL KMP_EXPAND_NAME(FTN_SET_NESTED)(int KMP_DEREF flag) {
 #ifdef KMP_STUB
   __kmps_set_nested(KMP_DEREF flag);
 #else

 
 int FTN_STDCALL KMP_EXPAND_NAME(FTN_GET_NUM_PLACES)(void) {
 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
   return 0;
 #else
   if (!TCR_4(__kmp_init_middle)) {
     __kmp_middle_initialize();
   }
-  __kmp_assign_root_init_mask();
   if (!KMP_AFFINITY_CAPABLE())
     return 0;
+  if (!__kmp_affin_reset) {
+    // only bind root here if its affinity reset is not requested
+    int gtid = __kmp_entry_gtid();
+    kmp_info_t *thread = __kmp_threads[gtid];
+    if (thread->th.th_team->t.t_level == 0) {
+      __kmp_assign_root_init_mask();
+    }
+  }
   return __kmp_affinity_num_masks;
 #endif
 }
 
 int FTN_STDCALL KMP_EXPAND_NAME(FTN_GET_PLACE_NUM_PROCS)(int place_num) {
 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
   return 0;
 #else
   int i;
   int retval = 0;
   if (!TCR_4(__kmp_init_middle)) {
     __kmp_middle_initialize();
   }
-  __kmp_assign_root_init_mask();
   if (!KMP_AFFINITY_CAPABLE())
     return 0;
+  if (!__kmp_affin_reset) {
+    // only bind root here if its affinity reset is not requested
+    int gtid = __kmp_entry_gtid();
+    kmp_info_t *thread = __kmp_threads[gtid];
+    if (thread->th.th_team->t.t_level == 0) {
+      __kmp_assign_root_init_mask();
+    }
+  }
   if (place_num < 0 || place_num >= (int)__kmp_affinity_num_masks)
     return 0;
   kmp_affin_mask_t *mask = KMP_CPU_INDEX(__kmp_affinity_masks, place_num);
   KMP_CPU_SET_ITERATE(i, mask) {
     if ((!KMP_CPU_ISSET(i, __kmp_affin_fullMask)) ||
         (!KMP_CPU_ISSET(i, mask))) {
       continue;
     }
     ++retval;
   }
   return retval;
 #endif
 }
 
 void FTN_STDCALL KMP_EXPAND_NAME(FTN_GET_PLACE_PROC_IDS)(int place_num,
                                                          int *ids) {
 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
 // Nothing.
 #else
   int i, j;
   if (!TCR_4(__kmp_init_middle)) {
     __kmp_middle_initialize();
   }
-  __kmp_assign_root_init_mask();
   if (!KMP_AFFINITY_CAPABLE())
     return;
+  if (!__kmp_affin_reset) {
+    // only bind root here if its affinity reset is not requested
+    int gtid = __kmp_entry_gtid();
+    kmp_info_t *thread = __kmp_threads[gtid];
+    if (thread->th.th_team->t.t_level == 0) {
+      __kmp_assign_root_init_mask();
+    }
+  }
   if (place_num < 0 || place_num >= (int)__kmp_affinity_num_masks)
     return;
   kmp_affin_mask_t *mask = KMP_CPU_INDEX(__kmp_affinity_masks, place_num);
   j = 0;
   KMP_CPU_SET_ITERATE(i, mask) {
     if ((!KMP_CPU_ISSET(i, __kmp_affin_fullMask)) ||
         (!KMP_CPU_ISSET(i, mask))) {
       continue;
     }
     ids[j++] = i;
   }
 #endif
 }
 
 int FTN_STDCALL KMP_EXPAND_NAME(FTN_GET_PLACE_NUM)(void) {
 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
   return -1;
 #else
   int gtid;
   kmp_info_t *thread;
   if (!TCR_4(__kmp_init_middle)) {
     __kmp_middle_initialize();
   }
-  __kmp_assign_root_init_mask();
   if (!KMP_AFFINITY_CAPABLE())
     return -1;
   gtid = __kmp_entry_gtid();
   thread = __kmp_thread_from_gtid(gtid);
+  if (thread->th.th_team->t.t_level == 0 && !__kmp_affin_reset) {
+    __kmp_assign_root_init_mask();
+  }
   if (thread->th.th_current_place < 0)
     return -1;
   return thread->th.th_current_place;
 #endif
 }
 
 int FTN_STDCALL KMP_EXPAND_NAME(FTN_GET_PARTITION_NUM_PLACES)(void) {
 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
   return 0;
 #else
   int gtid, num_places, first_place, last_place;
   kmp_info_t *thread;
   if (!TCR_4(__kmp_init_middle)) {
     __kmp_middle_initialize();
   }
-  __kmp_assign_root_init_mask();
   if (!KMP_AFFINITY_CAPABLE())
     return 0;
   gtid = __kmp_entry_gtid();
   thread = __kmp_thread_from_gtid(gtid);
+  if (thread->th.th_team->t.t_level == 0 && !__kmp_affin_reset) {
+    __kmp_assign_root_init_mask();
+  }
   first_place = thread->th.th_first_place;
   last_place = thread->th.th_last_place;
   if (first_place < 0 || last_place < 0)
     return 0;
   if (first_place <= last_place)
     num_places = last_place - first_place + 1;
   else
     num_places = __kmp_affinity_num_masks - first_place + last_place + 1;
   return num_places;
 #endif
 }
 
 void FTN_STDCALL
 KMP_EXPAND_NAME(FTN_GET_PARTITION_PLACE_NUMS)(int *place_nums) {
 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
 // Nothing.
 #else
   int i, gtid, place_num, first_place, last_place, start, end;
   kmp_info_t *thread;
   if (!TCR_4(__kmp_init_middle)) {
     __kmp_middle_initialize();
   }
-  __kmp_assign_root_init_mask();
   if (!KMP_AFFINITY_CAPABLE())
     return;
   gtid = __kmp_entry_gtid();
   thread = __kmp_thread_from_gtid(gtid);
+  if (thread->th.th_team->t.t_level == 0 && !__kmp_affin_reset) {
+    __kmp_assign_root_init_mask();
+  }
   first_place = thread->th.th_first_place;
   last_place = thread->th.th_last_place;
   if (first_place < 0 || last_place < 0)
     return;
   if (first_place <= last_place) {
     start = first_place;
     end = last_place;
   } else {

openmp/runtime/src/kmp_global.cpp

 int __kmp_affinity_verbose = FALSE;
 int __kmp_affinity_warnings = TRUE;
 int __kmp_affinity_respect_mask = affinity_respect_mask_default;
 char *__kmp_affinity_proclist = NULL;
 kmp_affin_mask_t *__kmp_affinity_masks = NULL;
 unsigned __kmp_affinity_num_masks = 0;
 
 char *__kmp_cpuinfo_file = NULL;
+bool __kmp_affin_reset = 0;
 
 #endif /* KMP_AFFINITY_SUPPORTED */
 
 kmp_nested_proc_bind_t __kmp_nested_proc_bind = {NULL, 0, 0};
 kmp_proc_bind_t __kmp_teams_proc_bind = proc_bind_spread;
 int __kmp_affinity_num_places = 0;
 int __kmp_display_affinity = FALSE;
 char *__kmp_affinity_format = NULL;

openmp/runtime/src/kmp_runtime.cpp

#endif
 
   // TODO: GEH - cannot do this assertion because root thread not set up as
   // executing
   // KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 0 );
   master_th->th.th_current_task->td_flags.executing = 1;
 
   __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
 
+#if KMP_AFFINITY_SUPPORTED
+  if (master_th->th.th_team->t.t_level == 0 && __kmp_affin_reset) {
+    __kmp_reset_root_init_mask(gtid);
+  }
+#endif
 #if OMPT_SUPPORT
   int flags =
       OMPT_INVOKER(fork_context) |
       ((team_microtask == (void *)__kmp_teams_master) ? ompt_parallel_league
                                                       : ompt_parallel_team);
   if (ompt_enabled.enabled) {
     __kmp_join_ompt(gtid, master_th, parent_team, parallel_data, flags,
                     codeptr);

openmp/runtime/src/kmp_settings.cpp

static void __kmp_parse_affinity_env(char const *name, char const *value,
   // Guards.
   int type = 0;
   int proclist = 0;
   int verbose = 0;
   int warnings = 0;
   int respect = 0;
   int gran = 0;
   int dups = 0;
+  int reset = 0;
   bool set = false;
 
   KMP_ASSERT(value != NULL);
 
   if (TCR_4(__kmp_init_middle)) {
     KMP_WARNING(EnvMiddleWarn, name);
     __kmp_env_toPrint(name, 0);
     return;
#define _set_param(_guard, _var, _val)                                         \
   }
 
 #define set_type(val) _set_param(type, *out_type, val)
 #define set_verbose(val) _set_param(verbose, *out_verbose, val)
 #define set_warnings(val) _set_param(warnings, *out_warn, val)
 #define set_respect(val) _set_param(respect, *out_respect, val)
 #define set_dups(val) _set_param(dups, *out_dups, val)
 #define set_proclist(val) _set_param(proclist, *out_proclist, val)
+#define set_reset(val) _set_param(reset, __kmp_affin_reset, val)
 
 #define set_gran(val, levels)                                                  \
   {                                                                            \
     if (gran == 0) {                                                           \
       *out_gran = val;                                                         \
       *out_gran_levels = levels;                                               \
     } else {                                                                   \
       EMIT_WARN(FALSE, (AffParamDefined, name, start));                        \
if (__kmp_match_str("none", buf, CCAST(const char **, &next))) {
       set_warnings(FALSE);
       buf = next;
     } else if (__kmp_match_str("respect", buf, CCAST(const char **, &next))) {
       set_respect(TRUE);
       buf = next;
     } else if (__kmp_match_str("norespect", buf, CCAST(const char **, &next))) {
       set_respect(FALSE);
       buf = next;
+    } else if (__kmp_match_str("reset", buf, CCAST(const char **, &next))) {
+      set_reset(TRUE);
+      buf = next;
+    } else if (__kmp_match_str("noreset", buf, CCAST(const char **, &next))) {
+      set_reset(FALSE);
+      buf = next;
     } else if (__kmp_match_str("duplicates", buf,
                                CCAST(const char **, &next)) ||
                __kmp_match_str("dups", buf, CCAST(const char **, &next))) {
       set_dups(TRUE);
       buf = next;
     } else if (__kmp_match_str("noduplicates", buf,
                                CCAST(const char **, &next)) ||
                __kmp_match_str("nodups", buf, CCAST(const char **, &next))) {

 
 #undef EMIT_WARN
 #undef _set_param
 #undef set_type
 #undef set_verbose
 #undef set_warnings
 #undef set_respect
 #undef set_granularity
+#undef set_reset
 
   __kmp_str_free(&buffer);
 
   if (proclist) {
     if (!type) {
       KMP_WARNING(AffProcListNoType, name);
       *out_type = affinity_explicit;
       __kmp_nested_proc_bind.bind_types[0] = proc_bind_intel;
if (__kmp_affinity_warnings) {
     __kmp_str_buf_print(buffer, "%s,", "nowarnings");
   }
   if (KMP_AFFINITY_CAPABLE()) {
     if (__kmp_affinity_respect_mask) {
       __kmp_str_buf_print(buffer, "%s,", "respect");
     } else {
       __kmp_str_buf_print(buffer, "%s,", "norespect");
     }
+    if (__kmp_affin_reset) {
+      __kmp_str_buf_print(buffer, "%s,", "reset");
+    } else {
+      __kmp_str_buf_print(buffer, "%s,", "noreset");
+    }
     __kmp_str_buf_print(buffer, "granularity=%s,",
                         __kmp_hw_get_keyword(__kmp_affinity_gran, false));
   }
   if (!KMP_AFFINITY_CAPABLE()) {
     __kmp_str_buf_print(buffer, "%s", "disabled");
   } else
     switch (__kmp_affinity_type) {
     case affinity_none:

openmp/runtime/test/affinity/kmp-affinity-reset.c

+// RUN: %libomp-compile -D_GNU_SOURCE
+// RUN: env OMP_NUM_THREADS=2,2 KMP_AFFINITY=reset,granularity=thread,compact %libomp-run
+// REQUIRES: linux
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <omp.h>
+#include "libomp_test_affinity.h"
+
+#define CHECK_EQUAL 0
+#define CHECK_NOT_EQUAL 1
+
+void check_primary_thread_affinity(int line, affinity_mask_t *other_aff,
+                                   int type) {
+  #pragma omp master
+  {
+    affinity_mask_t *primary_aff = affinity_mask_alloc();
+    get_thread_affinity(primary_aff);
+    if (type == CHECK_EQUAL && !affinity_mask_equal(primary_aff, other_aff)) {
+      fprintf(stderr, "error: line %d: primary affinity was not equal\n", line);
+      exit(EXIT_FAILURE);
+    } else if (type == CHECK_NOT_EQUAL &&
+               affinity_mask_equal(primary_aff, other_aff)) {
+      fprintf(stderr, "error: line %d: primary affinity was equal\n", line);
+      exit(EXIT_FAILURE);
+    }
+    affinity_mask_free(primary_aff);
+  }
+}
+
+#define CHECK_PRIMARY_THREAD_AFFINITY_EQUAL(other_aff)                         \
+  check_primary_thread_affinity(__LINE__, other_aff, CHECK_EQUAL)
+#define CHECK_PRIMARY_THREAD_AFFINITY_NOT_EQUAL(other_aff)                     \
+  check_primary_thread_affinity(__LINE__, other_aff, CHECK_NOT_EQUAL)
+
+int main() {
+  int i;
+  affinity_mask_t *initial_mask = affinity_mask_alloc();
+  get_thread_affinity(initial_mask);
+
+  for (i = 0; i < 10; ++i) {
+    #pragma omp parallel
+    {
+      CHECK_PRIMARY_THREAD_AFFINITY_NOT_EQUAL(initial_mask);
+    }
+    CHECK_PRIMARY_THREAD_AFFINITY_EQUAL(initial_mask);
+  }
+
+  omp_set_max_active_levels(2);
+  for (i = 0; i < 10; ++i) {
+    #pragma omp parallel
+    {
+      CHECK_PRIMARY_THREAD_AFFINITY_NOT_EQUAL(initial_mask);
+
+      #pragma omp parallel
+      CHECK_PRIMARY_THREAD_AFFINITY_NOT_EQUAL(initial_mask);
+
+      CHECK_PRIMARY_THREAD_AFFINITY_NOT_EQUAL(initial_mask);
+    }
+    CHECK_PRIMARY_THREAD_AFFINITY_EQUAL(initial_mask);
+  }
+
+  affinity_mask_free(initial_mask);
+  return EXIT_SUCCESS;
+}