[OpenMP] libomp: Add masked construct, change proc-bind keyword, deprecate master construct

[tlwilmar]

Summary

Added API for "masked" construct via two entrypoints: kmpc_masked, and kmpc_end_masked. The "master" construct is deprecated. Changed proc-bind keyword from "master" to "primary". Use of both master construct and master as proc-bind keyword is still allowed, but deprecated.

Remove references to "master" in comments and strings, and replace with "primary" or "primary thread". Function names and variables were not touched, nor were references to deprecated master construct. These can be updated over time. No new code should refer to master.

[jlpeyton]

openmp/runtime/src/kmp.h

1455

The corresponding string "\"masked\"" should be put into cons_text_c in kmp_error_cpp

[tlwilmar]

Added masked to cons_text_c.

[jlpeyton]

LGTM

[tlwilmar]

Hi Johnny,
When I just commit via the instructions on our wiki, I just see this separate e-mail labeled "[Diffusion]" which looks like a separate review in Phabricator. When others push their changes, it appears attached to the original phabricator review, and seems to close it automagically. What am I missing there?

Thanks!

[jlpeyton]

In D97019#2606795, @tlwilmar wrote:

Hi Johnny,
When I just commit via the instructions on our wiki, I just see this separate e-mail labeled "[Diffusion]" which looks like a separate review in Phabricator. When others push their changes, it appears attached to the original phabricator review, and seems to close it automagically. What am I missing there?

Thanks!

Ah, you have to include the line:
Differential Revision: https://reviews.llvm.org/D97019
in the commit message. This line is used to attach the commit to the Phabricator review.

[jlpeyton]

@tlwilmar , you can just close this revision manually, but if you look at the git log for the llvm-project you'll see the "Differential Revision: <hyperlink>" line in many of the commits.

openmp/runtime/src/dllexports

# USED ABOVE __kmpc_critical_with_hint    270
         __kmpc_omp_reg_task_with_affinity   272
         __kmpc_pause_resource               273
         __kmpc_task_reduction_modifier_init 274
         __kmpc_task_reduction_modifier_fini 275
         __kmpc_task_allow_completion_event  276
         __kmpc_taskred_init                 277
         __kmpc_taskred_modifier_init        278
         __kmpc_omp_target_task_alloc        279
+        __kmpc_masked                       282
+        __kmpc_end_masked                   283
 %endif
 
 # User API entry points that have both lower- and upper- case versions for Fortran.
 # Number for lowercase version is indicated.  Number for uppercase is obtained by adding 1000.
 # User API entry points are entry points that start with 'kmp_' or 'omp_'.
 
 omp_destroy_lock                            700
 omp_destroy_nest_lock                       701

openmp/runtime/src/i18n/en_US.txt

 
 Info                         "OMP: Info #%1$d: %2$s\n"
 Warning                      "OMP: Warning #%1$d: %2$s\n"
 Fatal                        "OMP: Error #%1$d: %2$s\n"
 SysErr                       "OMP: System error #%1$d: %2$s\n"
 Hint                         "OMP: Hint %1$s\n"
 
 Pragma                       "%1$s pragma (at %2$s:%3$s():%4$s)"
-    # %1 is pragma name (like "parallel" or "master",
+    # %1 is pragma name (like "parallel" or "masked",
     # %2 is file name,
     # %3 is function (routine) name,
     # %4 is the line number (as string, so "s" type specifier should be used).
 
 
 
 # --------------------------------------------------------------------------------------------------
 -*- MESSAGES -*-

openmp/runtime/src/kmp.h

 extern char *__kmp_cpuinfo_file;
 
 #endif /* KMP_AFFINITY_SUPPORTED */
 
 // This needs to be kept in sync with the values in omp.h !!!
 typedef enum kmp_proc_bind_t {
   proc_bind_false = 0,
   proc_bind_true,
-  proc_bind_master,
+  proc_bind_primary,
   proc_bind_close,
   proc_bind_spread,
   proc_bind_intel, // use KMP_AFFINITY interface
   proc_bind_default
 } kmp_proc_bind_t;
 
 typedef struct kmp_nested_proc_bind_t {
   kmp_proc_bind_t *bind_types;
enum cons_type {
   ct_pdo_ordered,
   ct_psections,
   ct_psingle,
   ct_critical,
   ct_ordered_in_parallel,
   ct_ordered_in_pdo,
   ct_master,
   ct_reduce,
-  ct_barrier
+  ct_barrier,
+  ct_masked
 };
 
 #define IS_CONS_TYPE_ORDERED(ct) ((ct) == ct_pdo_ordered)
 
 struct cons_data {
   ident_t const *ident;
   enum cons_type type;
   int prev;
struct private_data {
   int more; /* The repeat count for this descriptor */
   size_t size; /* The data size for this descriptor    */
 };
 
 struct private_common {
   struct private_common *next;
   struct private_common *link;
   void *gbl_addr;
-  void *par_addr; /* par_addr == gbl_addr for MASTER thread */
+  void *par_addr; /* par_addr == gbl_addr for PRIMARY thread */
   size_t cmn_size;
 };
 
 struct shared_common {
   struct shared_common *next;
   struct private_data *pod_init;
   void *obj_init;
   void *gbl_addr;

 /* Team barrier needs only non-volatile arrived counter */
 union KMP_ALIGN_CACHE kmp_barrier_team_union {
   double b_align; /* use worst case alignment */
   char b_pad[CACHE_LINE];
   struct {
     kmp_uint64 b_arrived; /* STATE => task reached synch point. */
 #if USE_DEBUGGER
     // The following two fields are indended for the debugger solely. Only
-    // master of the team accesses these fields: the first one is increased by
-    // 1 when master arrives to a barrier, the second one is increased by one
-    // when all the threads arrived.
+    // primary thread of the team accesses these fields: the first one is
+    // increased by 1 when the primary thread arrives to a barrier, the second
+    // one is increased by one when all the threads arrived.
     kmp_uint b_master_arrived;
     kmp_uint b_team_arrived;
 #endif
   };
 };
 
 typedef union kmp_barrier_team_union kmp_balign_team_t;
 

 #endif
 typedef struct kmp_teams_size {
   kmp_int32 nteams; // number of teams in a league
   kmp_int32 nth; // number of threads in each team of the league
 } kmp_teams_size_t;
 
 // This struct stores a thread that acts as a "root" for a contention
 // group. Contention groups are rooted at kmp_root threads, but also at
-// each master thread of each team created in the teams construct.
+// each primary thread of each team created in the teams construct.
 // This struct therefore also stores a thread_limit associated with
 // that contention group, and a counter to track the number of threads
 // active in that contention group. Each thread has a list of these: CG
 // root threads have an entry in their list in which cg_root refers to
 // the thread itself, whereas other workers in the CG will have a
 // single entry where cg_root is same as the entry containing their CG
 // root. When a thread encounters a teams construct, it will add a new
 // entry to the front of its list, because it now roots a new CG.
 typedef struct kmp_cg_root {
   kmp_info_p *cg_root; // "root" thread for a contention group
   // The CG root's limit comes from OMP_THREAD_LIMIT for root threads, or
-  // thread_limit clause for teams masters
+  // thread_limit clause for teams primary threads
   kmp_int32 cg_thread_limit;
   kmp_int32 cg_nthreads; // Count of active threads in CG rooted at cg_root
   struct kmp_cg_root *up; // pointer to higher level CG root in list
 } kmp_cg_root_t;
 
 // OpenMP thread data structures
 
 typedef struct KMP_ALIGN_CACHE kmp_base_info {
   /* Start with the readonly data which is cache aligned and padded. This is
-     written before the thread starts working by the master. Uber masters may
-     update themselves later. Usage does not consider serialized regions.  */
+     written before the thread starts working by the primary thread. Uber
+     masters may update themselves later. Usage does not consider serialized
+     regions.  */
   kmp_desc_t th_info;
   kmp_team_p *th_team; /* team we belong to */
   kmp_root_p *th_root; /* pointer to root of task hierarchy */
   kmp_info_p *th_next_pool; /* next available thread in the pool */
   kmp_disp_t *th_dispatch; /* thread's dispatch data */
   int th_in_pool; /* in thread pool (32 bits for TCR/TCW) */
 
   /* The following are cached from the team info structure */
   /* TODO use these in more places as determined to be needed via profiling */
   int th_team_nproc; /* number of threads in a team */
-  kmp_info_p *th_team_master; /* the team's master thread */
+  kmp_info_p *th_team_master; /* the team's primary thread */
   int th_team_serialized; /* team is serialized */
   microtask_t th_teams_microtask; /* save entry address for teams construct */
   int th_teams_level; /* save initial level of teams construct */
 /* it is 0 on device but may be any on host */
 
 /* The blocktime info is copied from the team struct to the thread struct */
 /* at the start of a barrier, and the values stored in the team are used  */
 /* at points in the code where the team struct is no longer guaranteed    */
 /* to exist (from the POV of worker threads).                             */
 #if KMP_USE_MONITOR
   int th_team_bt_intervals;
   int th_team_bt_set;
 #else
   kmp_uint64 th_team_bt_intervals;
 #endif
 
 #if KMP_AFFINITY_SUPPORTED
   kmp_affin_mask_t *th_affin_mask; /* thread's current affinity mask */
 #endif
   omp_allocator_handle_t th_def_allocator; /* default allocator */
-  /* The data set by the master at reinit, then R/W by the worker */
+  /* The data set by the primary thread at reinit, then R/W by the worker */
   KMP_ALIGN_CACHE int
       th_set_nproc; /* if > 0, then only use this request for the next fork */
 #if KMP_NESTED_HOT_TEAMS
   kmp_hot_team_ptr_t *th_hot_teams; /* array of hot teams */
 #endif
   kmp_proc_bind_t
       th_set_proc_bind; /* if != proc_bind_default, use request for next fork */
   kmp_teams_size_t
/* TODO the first serial team should actually be stored in the info_t
      structure.  this will help reduce initial allocation overhead */
   KMP_ALIGN_CACHE kmp_team_p
       *th_serial_team; /*serialized team held in reserve*/
 
 #if OMPT_SUPPORT
   ompt_thread_info_t ompt_thread_info;
 #endif
 
-  /* The following are also read by the master during reinit */
+  /* The following are also read by the primary thread during reinit */
   struct common_table *th_pri_common;
 
   volatile kmp_uint32 th_spin_here; /* thread-local location for spinning */
   /* while awaiting queuing lock acquire */
 
   volatile void *th_sleep_loc; // this points at a kmp_flag<T>
 
   ident_t *th_ident;

 
 typedef int (*launch_t)(int gtid);
 
 /* Minimum number of ARGV entries to malloc if necessary */
 #define KMP_MIN_MALLOC_ARGV_ENTRIES 100
 
 // Set up how many argv pointers will fit in cache lines containing
 // t_inline_argv. Historically, we have supported at least 96 bytes. Using a
-// larger value for more space between the master write/worker read section and
+// larger value for more space between the primary write/worker read section and
 // read/write by all section seems to buy more performance on EPCC PARALLEL.
 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
 #define KMP_INLINE_ARGV_BYTES                                                  \
   (4 * CACHE_LINE -                                                            \
    ((3 * KMP_PTR_SKIP + 2 * sizeof(int) + 2 * sizeof(kmp_int8) +               \
      sizeof(kmp_int16) + sizeof(kmp_uint32)) %                                 \
     CACHE_LINE))
 #else
typedef struct KMP_ALIGN_CACHE kmp_base_team {
   kmp_balign_team_t t_bar[bs_last_barrier];
   std::atomic<int> t_construct; // count of single directive encountered by team
   char pad[sizeof(kmp_lock_t)]; // padding to maintain performance on big iron
 
   // [0] - parallel / [1] - worksharing task reduction data shared by taskgroups
   std::atomic<void *> t_tg_reduce_data[2]; // to support task modifier
   std::atomic<int> t_tg_fini_counter[2]; // sync end of task reductions
 
-  // Master only
+  // Primary thread only
   // ---------------------------------------------------------------------------
-  KMP_ALIGN_CACHE int t_master_tid; // tid of master in parent team
-  int t_master_this_cons; // "this_construct" single counter of master in parent
-  // team
+  KMP_ALIGN_CACHE int t_master_tid; // tid of primary thread in parent team
+  int t_master_this_cons; // "this_construct" single counter of primary thread
+  // in parent team
   ident_t *t_ident; // if volatile, have to change too much other crud to
   // volatile too
   kmp_team_p *t_parent; // parent team
   kmp_team_p *t_next_pool; // next free team in the team pool
   kmp_disp_t *t_dispatch; // thread's dispatch data
   kmp_task_team_t *t_task_team[2]; // Task team struct; switch between 2
   kmp_proc_bind_t t_proc_bind; // bind type for par region
 #if USE_ITT_BUILD
   kmp_uint64 t_region_time; // region begin timestamp
 #endif /* USE_ITT_BUILD */
 
-  // Master write, workers read
+  // Primary thread write, workers read
   // --------------------------------------------------------------------------
   KMP_ALIGN_CACHE void **t_argv;
   int t_argc;
   int t_nproc; // number of threads in team
   microtask_t t_pkfn;
   launch_t t_invoke; // procedure to launch the microtask
 
 #if OMPT_SUPPORT
#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
   int t_max_nproc; // max threads this team can handle (dynamically expandable)
   int t_serialized; // levels deep of serialized teams
   dispatch_shared_info_t *t_disp_buffer; // buffers for dispatch system
   int t_id; // team's id, assigned by debugger.
   int t_active_level; // nested active parallel level
   kmp_r_sched_t t_sched; // run-time schedule for the team
 #if KMP_AFFINITY_SUPPORTED
   int t_first_place; // first & last place in parent thread's partition.
-  int t_last_place; // Restore these values to master after par region.
+  int t_last_place; // Restore these values to primary thread after par region.
 #endif // KMP_AFFINITY_SUPPORTED
   int t_display_affinity;
   int t_size_changed; // team size was changed?: 0: no, 1: yes, -1: changed via
   // omp_set_num_threads() call
   omp_allocator_handle_t t_def_allocator; /* default allocator */
 
 // Read/write by workers as well
 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64)

 
 KMP_EXPORT void __kmpc_serialized_parallel(ident_t *, kmp_int32 global_tid);
 KMP_EXPORT void __kmpc_end_serialized_parallel(ident_t *, kmp_int32 global_tid);
 
 KMP_EXPORT void __kmpc_flush(ident_t *);
 KMP_EXPORT void __kmpc_barrier(ident_t *, kmp_int32 global_tid);
 KMP_EXPORT kmp_int32 __kmpc_master(ident_t *, kmp_int32 global_tid);
 KMP_EXPORT void __kmpc_end_master(ident_t *, kmp_int32 global_tid);
+KMP_EXPORT kmp_int32 __kmpc_masked(ident_t *, kmp_int32 global_tid,
+                                   kmp_int32 filter);
+KMP_EXPORT void __kmpc_end_masked(ident_t *, kmp_int32 global_tid);
 KMP_EXPORT void __kmpc_ordered(ident_t *, kmp_int32 global_tid);
 KMP_EXPORT void __kmpc_end_ordered(ident_t *, kmp_int32 global_tid);
 KMP_EXPORT void __kmpc_critical(ident_t *, kmp_int32 global_tid,
                                 kmp_critical_name *);
 KMP_EXPORT void __kmpc_end_critical(ident_t *, kmp_int32 global_tid,
                                     kmp_critical_name *);
 KMP_EXPORT void __kmpc_critical_with_hint(ident_t *, kmp_int32 global_tid,
                                           kmp_critical_name *, uint32_t hint);

openmp/runtime/src/kmp_barrier.cpp

#endif
   // arrived.
   if (!KMP_MASTER_TID(tid)) {
     KA_TRACE(20,
              ("__kmp_linear_barrier_gather: T#%d(%d:%d) releasing T#%d(%d:%d)"
               "arrived(%p): %llu => %llu\n",
               gtid, team->t.t_id, tid, __kmp_gtid_from_tid(0, team),
               team->t.t_id, 0, &thr_bar->b_arrived, thr_bar->b_arrived,
               thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
-    // Mark arrival to master thread
+    // Mark arrival to primary thread
     /* After performing this write, a worker thread may not assume that the team
-       is valid any more - it could be deallocated by the master thread at any
+       is valid any more - it could be deallocated by the primary thread at any
        time. */
     ANNOTATE_BARRIER_BEGIN(this_thr);
     kmp_flag_64<> flag(&thr_bar->b_arrived, other_threads[0]);
     flag.release();
   } else {
     kmp_balign_team_t *team_bar = &team->t.t_bar[bt];
     int nproc = this_thr->th.th_team_nproc;
     int i;
if (KMP_MASTER_TID(tid)) {
     unsigned int i;
     kmp_uint32 nproc = this_thr->th.th_team_nproc;
     kmp_info_t **other_threads;
 
     team = __kmp_threads[gtid]->th.th_team;
     KMP_DEBUG_ASSERT(team != NULL);
     other_threads = team->t.t_threads;
 
-    KA_TRACE(20, ("__kmp_linear_barrier_release: T#%d(%d:%d) master enter for "
+    KA_TRACE(20, ("__kmp_linear_barrier_release: T#%d(%d:%d) primary enter for "
                   "barrier type %d\n",
                   gtid, team->t.t_id, tid, bt));
 
     if (nproc > 1) {
 #if KMP_BARRIER_ICV_PUSH
       {
         KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_icv_copy);
         if (propagate_icvs) {
#endif /* KMP_CACHE_MANAGE */
              other_threads[i]->th.th_bar[bt].bb.b_go,
              other_threads[i]->th.th_bar[bt].bb.b_go + KMP_BARRIER_STATE_BUMP));
         ANNOTATE_BARRIER_BEGIN(other_threads[i]);
         kmp_flag_64<> flag(&other_threads[i]->th.th_bar[bt].bb.b_go,
                          other_threads[i]);
         flag.release();
       }
     }
-  } else { // Wait for the MASTER thread to release us
+  } else { // Wait for the PRIMARY thread to release us
     KA_TRACE(20, ("__kmp_linear_barrier_release: T#%d wait go(%p) == %u\n",
                   gtid, &thr_bar->b_go, KMP_BARRIER_STATE_BUMP));
     if (cancellable) {
       kmp_flag_64<true, false> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
       if (flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj)))
         return true;
     } else {
       kmp_flag_64<> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
KA_TRACE(20,
              ("__kmp_tree_barrier_gather: T#%d(%d:%d) releasing T#%d(%d:%d) "
               "arrived(%p): %llu => %llu\n",
               gtid, team->t.t_id, tid, __kmp_gtid_from_tid(parent_tid, team),
               team->t.t_id, parent_tid, &thr_bar->b_arrived, thr_bar->b_arrived,
               thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
 
     // Mark arrival to parent thread
     /* After performing this write, a worker thread may not assume that the team
-       is valid any more - it could be deallocated by the master thread at any
+       is valid any more - it could be deallocated by the primary thread at any
        time.  */
     ANNOTATE_BARRIER_BEGIN(this_thr);
     kmp_flag_64<> flag(&thr_bar->b_arrived, other_threads[parent_tid]);
     flag.release();
   } else {
-    // Need to update the team arrived pointer if we are the master thread
+    // Need to update the team arrived pointer if we are the primary thread
     if (nproc > 1) // New value was already computed above
       team->t.t_bar[bt].b_arrived = new_state;
     else
       team->t.t_bar[bt].b_arrived += KMP_BARRIER_STATE_BUMP;
     KA_TRACE(20, ("__kmp_tree_barrier_gather: T#%d(%d:%d) set team %d "
                   "arrived(%p) = %llu\n",
                   gtid, team->t.t_id, tid, team->t.t_id,
                   &team->t.t_bar[bt].b_arrived, team->t.t_bar[bt].b_arrived));
#endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */
     TCW_4(thr_bar->b_go, KMP_INIT_BARRIER_STATE);
     KA_TRACE(20,
              ("__kmp_tree_barrier_release: T#%d(%d:%d) set go(%p) = %u\n", gtid,
               team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE));
     KMP_MB(); // Flush all pending memory write invalidates.
   } else {
     team = __kmp_threads[gtid]->th.th_team;
     KMP_DEBUG_ASSERT(team != NULL);
-    KA_TRACE(20, ("__kmp_tree_barrier_release: T#%d(%d:%d) master enter for "
+    KA_TRACE(20, ("__kmp_tree_barrier_release: T#%d(%d:%d) primary enter for "
                   "barrier type %d\n",
                   gtid, team->t.t_id, tid, bt));
   }
   nproc = this_thr->th.th_team_nproc;
   child_tid = (tid << branch_bits) + 1;
 
   if (child_tid < nproc) {
     kmp_info_t **other_threads = team->t.t_threads;
if (((tid >> level) & (branch_factor - 1)) != 0) {
                 "arrived(%p): %llu => %llu\n",
                 gtid, team->t.t_id, tid, __kmp_gtid_from_tid(parent_tid, team),
                 team->t.t_id, parent_tid, &thr_bar->b_arrived,
                 thr_bar->b_arrived,
                 thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
       // Mark arrival to parent thread
       /* After performing this write (in the last iteration of the enclosing for
          loop), a worker thread may not assume that the team is valid any more
-         - it could be deallocated by the master thread at any time.  */
+         - it could be deallocated by the primary thread at any time.  */
       ANNOTATE_BARRIER_BEGIN(this_thr);
       p_flag.set_waiter(other_threads[parent_tid]);
       p_flag.release();
       break;
     }
 
     // Parent threads wait for children to arrive
     if (new_state == KMP_BARRIER_UNUSED_STATE)
#endif
         OMPT_REDUCTION_END;
         ANNOTATE_REDUCE_BEFORE(reduce);
         ANNOTATE_REDUCE_BEFORE(&team->t.t_bar);
       }
     }
   }
 
   if (KMP_MASTER_TID(tid)) {
-    // Need to update the team arrived pointer if we are the master thread
+    // Need to update the team arrived pointer if we are the primary thread
     if (new_state == KMP_BARRIER_UNUSED_STATE)
       team->t.t_bar[bt].b_arrived += KMP_BARRIER_STATE_BUMP;
     else
       team->t.t_bar[bt].b_arrived = new_state;
     KA_TRACE(20, ("__kmp_hyper_barrier_gather: T#%d(%d:%d) set team %d "
                   "arrived(%p) = %llu\n",
                   gtid, team->t.t_id, tid, team->t.t_id,
                   &team->t.t_bar[bt].b_arrived, team->t.t_bar[bt].b_arrived));
static void __kmp_hyper_barrier_release(
   kmp_uint32 child_tid;
   kmp_uint32 offset;
   kmp_uint32 level;
 
   /* Perform a hypercube-embedded tree release for all of the threads that have
      been gathered. If KMP_REVERSE_HYPER_BAR is defined (default) the threads
      are released in the reverse order of the corresponding gather, otherwise
      threads are released in the same order. */
-  if (KMP_MASTER_TID(tid)) { // master
+  if (KMP_MASTER_TID(tid)) { // primary thread
     team = __kmp_threads[gtid]->th.th_team;
     KMP_DEBUG_ASSERT(team != NULL);
-    KA_TRACE(20, ("__kmp_hyper_barrier_release: T#%d(%d:%d) master enter for "
+    KA_TRACE(20, ("__kmp_hyper_barrier_release: T#%d(%d:%d) primary enter for "
                   "barrier type %d\n",
                   gtid, team->t.t_id, tid, bt));
 #if KMP_BARRIER_ICV_PUSH
-    if (propagate_icvs) { // master already has ICVs in final destination; copy
+    if (propagate_icvs) { // primary already has ICVs in final destination; copy
       copy_icvs(&thr_bar->th_fixed_icvs,
                 &team->t.t_implicit_task_taskdata[tid].td_icvs);
     }
 #endif
   } else { // Handle fork barrier workers who aren't part of a team yet
     KA_TRACE(20, ("__kmp_hyper_barrier_release: T#%d wait go(%p) == %u\n", gtid,
                   &thr_bar->b_go, KMP_BARRIER_STATE_BUMP));
     // Wait for parent thread to release us
static bool __kmp_init_hierarchical_barrier_thread(enum barrier_type bt,
   bool tid_changed = tid != thr_bar->old_tid;
   bool retval = false;
 
   if (uninitialized || team_sz_changed) {
     __kmp_get_hierarchy(nproc, thr_bar);
   }
 
   if (uninitialized || team_sz_changed || tid_changed) {
-    thr_bar->my_level = thr_bar->depth - 1; // default for master
-    thr_bar->parent_tid = -1; // default for master
-    if (!KMP_MASTER_TID(
-            tid)) { // if not master, find parent thread in hierarchy
+    thr_bar->my_level = thr_bar->depth - 1; // default for primary thread
+    thr_bar->parent_tid = -1; // default for primary thread
+    if (!KMP_MASTER_TID(tid)) {
+      // if not primary thread, find parent thread in hierarchy
       kmp_uint32 d = 0;
       while (d < thr_bar->depth) { // find parent based on level of thread in
         // hierarchy, and note level
         kmp_uint32 rem;
-        if (d == thr_bar->depth - 2) { // reached level right below the master
+        if (d == thr_bar->depth - 2) { // reached level right below the primary
           thr_bar->parent_tid = 0;
           thr_bar->my_level = d;
           break;
         } else if ((rem = tid % thr_bar->skip_per_level[d + 1]) != 0) {
           // TODO: can we make the above op faster?
           // thread is not a subtree root at next level, so this is max
           thr_bar->parent_tid = tid - rem;
           thr_bar->my_level = d;
if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
                       child_thr->th.th_local.reduce_data);
             ANNOTATE_REDUCE_BEFORE(reduce);
             ANNOTATE_REDUCE_BEFORE(&team->t.t_bar);
           }
         }
       }
     }
   }
-  // All subordinates are gathered; now release parent if not master thread
+  // All subordinates are gathered; now release parent if not primary thread
 
   if (!KMP_MASTER_TID(tid)) { // worker threads release parent in hierarchy
     KA_TRACE(20, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) releasing"
                   " T#%d(%d:%d) arrived(%p): %llu => %llu\n",
                   gtid, team->t.t_id, tid,
                   __kmp_gtid_from_tid(thr_bar->parent_tid, team), team->t.t_id,
                   thr_bar->parent_tid, &thr_bar->b_arrived, thr_bar->b_arrived,
                   thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
     /* Mark arrival to parent: After performing this write, a worker thread may
        not assume that the team is valid any more - it could be deallocated by
-       the master thread at any time. */
+       the primary thread at any time. */
     if (thr_bar->my_level || __kmp_dflt_blocktime != KMP_MAX_BLOCKTIME ||
         !thr_bar->use_oncore_barrier) { // Parent is waiting on my b_arrived
       // flag; release it
       ANNOTATE_BARRIER_BEGIN(this_thr);
       kmp_flag_64<> flag(&thr_bar->b_arrived,
                          other_threads[thr_bar->parent_tid]);
       flag.release();
     } else {
       // Leaf does special release on "offset" bits of parent's b_arrived flag
       thr_bar->b_arrived = team->t.t_bar[bt].b_arrived + KMP_BARRIER_STATE_BUMP;
       kmp_flag_oncore flag(&thr_bar->parent_bar->b_arrived,
                            thr_bar->offset + 1);
       flag.set_waiter(other_threads[thr_bar->parent_tid]);
       flag.release();
     }
-  } else { // Master thread needs to update the team's b_arrived value
+  } else { // Primary thread needs to update the team's b_arrived value
     team->t.t_bar[bt].b_arrived = new_state;
     KA_TRACE(20, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) set team %d "
                   "arrived(%p) = %llu\n",
                   gtid, team->t.t_id, tid, team->t.t_id,
                   &team->t.t_bar[bt].b_arrived, team->t.t_bar[bt].b_arrived));
   }
   // Is the team access below unsafe or just technically invalid?
   KA_TRACE(20, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) exit for "
                 "barrier type %d\n",
                 gtid, team->t.t_id, tid, bt));
 }
 
 static void __kmp_hierarchical_barrier_release(
     enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
     int propagate_icvs USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
   KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_hier_release);
   kmp_team_t *team;
   kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
   kmp_uint32 nproc;
   bool team_change = false; // indicates on-core barrier shouldn't be used
 
   if (KMP_MASTER_TID(tid)) {
     team = __kmp_threads[gtid]->th.th_team;
     KMP_DEBUG_ASSERT(team != NULL);
-    KA_TRACE(20, ("__kmp_hierarchical_barrier_release: T#%d(%d:%d) master "
+    KA_TRACE(20, ("__kmp_hierarchical_barrier_release: T#%d(%d:%d) primary "
                   "entered barrier type %d\n",
                   gtid, team->t.t_id, tid, bt));
   } else { // Worker threads
     // Wait for parent thread to release me
     if (!thr_bar->use_oncore_barrier ||
         __kmp_dflt_blocktime != KMP_MAX_BLOCKTIME || thr_bar->my_level != 0 ||
         thr_bar->team == NULL) {
       // Use traditional method of waiting on my own b_go flag
static void __kmp_hierarchical_barrier_release(
   if (team_change)
     old_leaf_kids = 0;
 
 #if KMP_BARRIER_ICV_PUSH
   if (propagate_icvs) {
     __kmp_init_implicit_task(team->t.t_ident, team->t.t_threads[tid], team, tid,
                              FALSE);
     if (KMP_MASTER_TID(
-            tid)) { // master already has copy in final destination; copy
+            tid)) { // primary already has copy in final destination; copy
       copy_icvs(&thr_bar->th_fixed_icvs,
                 &team->t.t_implicit_task_taskdata[tid].td_icvs);
     } else if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
                thr_bar->use_oncore_barrier) { // optimization for inf blocktime
       if (!thr_bar->my_level) // I'm a leaf in the hierarchy (my_level==0)
         // leaves (on-core children) pull parent's fixed ICVs directly to local
         // ICV store
         copy_icvs(&team->t.t_implicit_task_taskdata[tid].td_icvs,
template <> struct is_cancellable<false> {
   constexpr operator bool() const { return false; }
 };
 
 // Internal function to do a barrier.
 /* If is_split is true, do a split barrier, otherwise, do a plain barrier
    If reduce is non-NULL, do a split reduction barrier, otherwise, do a split
    barrier
    When cancellable = false,
-     Returns 0 if master thread, 1 if worker thread.
+     Returns 0 if primary thread, 1 if worker thread.
    When cancellable = true
      Returns 0 if not cancelled, 1 if cancelled.  */
 template <bool cancellable = false>
 static int __kmp_barrier_template(enum barrier_type bt, int gtid, int is_split,
                                   size_t reduce_size, void *reduce_data,
                                   void (*reduce)(void *, void *)) {
   KMP_TIME_PARTITIONED_BLOCK(OMP_plain_barrier);
   KMP_SET_THREAD_STATE_BLOCK(PLAIN_BARRIER);
#endif
     }
 
 #if USE_ITT_BUILD
     if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
       __kmp_itt_barrier_starting(gtid, itt_sync_obj);
 #endif /* USE_ITT_BUILD */
 #if USE_DEBUGGER
     // Let the debugger know: the thread arrived to the barrier and waiting.
-    if (KMP_MASTER_TID(tid)) { // Master counter is stored in team structure.
+    if (KMP_MASTER_TID(tid)) { // Primary thread counter stored in team struct
       team->t.t_bar[bt].b_master_arrived += 1;
     } else {
       this_thr->th.th_bar[bt].bb.b_worker_arrived += 1;
     } // if
 #endif /* USE_DEBUGGER */
     if (reduce != NULL) {
       // KMP_DEBUG_ASSERT( is_split == TRUE );  // #C69956
       this_thr->th.th_local.reduce_data = reduce_data;
#endif
         kmp_int32 cancel_request = KMP_ATOMIC_LD_RLX(&team->t.t_cancel_request);
         // Reset cancellation flag for worksharing constructs
         if (cancel_request == cancel_loop ||
             cancel_request == cancel_sections) {
           KMP_ATOMIC_ST_RLX(&team->t.t_cancel_request, cancel_noreq);
         }
       }
 #if USE_ITT_BUILD
-      /* TODO: In case of split reduction barrier, master thread may send
+      /* TODO: In case of split reduction barrier, primary thread may send
          acquired event early, before the final summation into the shared
          variable is done (final summation can be a long operation for array
          reductions).  */
       if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
         __kmp_itt_barrier_middle(gtid, itt_sync_obj);
 #endif /* USE_ITT_BUILD */
 #if USE_ITT_BUILD && USE_ITT_NOTIFY
       // Barrier - report frame end (only if active_level == 1)
#if USE_ITT_BUILD && USE_ITT_NOTIFY
           break;
         case 2: // AC 2015-01-19: currently does not work for hierarchical (to
           // be fixed)
           __kmp_itt_frame_submit(gtid, this_thr->th.th_bar_min_time, cur_time,
                                  1, loc, nproc);
           break;
         case 3:
           if (__itt_metadata_add_ptr) {
-            // Initialize with master's wait time
+            // Initialize with primary thread's wait time
             kmp_uint64 delta = cur_time - this_thr->th.th_bar_arrive_time;
             // Set arrive time to zero to be able to check it in
             // __kmp_invoke_task(); the same is done inside the loop below
             this_thr->th.th_bar_arrive_time = 0;
             for (i = 1; i < nproc; ++i) {
               delta += (cur_time - other_threads[i]->th.th_bar_arrive_time);
               other_threads[i]->th.th_bar_arrive_time = 0;
             }

 #endif
   ANNOTATE_BARRIER_END(&team->t.t_bar);
 
   if (cancellable)
     return (int)cancelled;
   return status;
 }
 
-// Returns 0 if master thread, 1 if worker thread.
+// Returns 0 if primary thread, 1 if worker thread.
 int __kmp_barrier(enum barrier_type bt, int gtid, int is_split,
                   size_t reduce_size, void *reduce_data,
                   void (*reduce)(void *, void *)) {
   return __kmp_barrier_template<>(bt, gtid, is_split, reduce_size, reduce_data,
                                   reduce);
 }
 
 #if defined(KMP_GOMP_COMPAT)
 // Returns 1 if cancelled, 0 otherwise
 int __kmp_barrier_gomp_cancel(int gtid) {
   if (__kmp_omp_cancellation) {
     int cancelled = __kmp_barrier_template<true>(bs_plain_barrier, gtid, FALSE,
                                                  0, NULL, NULL);
     if (cancelled) {
       int tid = __kmp_tid_from_gtid(gtid);
       kmp_info_t *this_thr = __kmp_threads[gtid];
       if (KMP_MASTER_TID(tid)) {
-        // Master does not need to revert anything
+        // Primary thread does not need to revert anything
       } else {
         // Workers need to revert their private b_arrived flag
         this_thr->th.th_bar[bs_plain_barrier].bb.b_arrived -=
             KMP_BARRIER_STATE_BUMP;
       }
     }
     return cancelled;
   }
#endif /* USE_ITT_BUILD */
   }
   default: {
     __kmp_linear_barrier_gather(bs_forkjoin_barrier, this_thr, gtid, tid,
                                 NULL USE_ITT_BUILD_ARG(itt_sync_obj));
   }
   }
 
   /* From this point on, the team data structure may be deallocated at any time
-     by the master thread - it is unsafe to reference it in any of the worker
+     by the primary thread - it is unsafe to reference it in any of the worker
      threads. Any per-team data items that need to be referenced before the
      end of the barrier should be moved to the kmp_task_team_t structs.  */
   if (KMP_MASTER_TID(tid)) {
     if (__kmp_tasking_mode != tskm_immediate_exec) {
       __kmp_task_team_wait(this_thr, team USE_ITT_BUILD_ARG(itt_sync_obj));
     }
     if (__kmp_display_affinity) {
       KMP_CHECK_UPDATE(team->t.t_display_affinity, 0);
     }
 #if KMP_STATS_ENABLED
-    // Have master thread flag the workers to indicate they are now waiting for
+    // Have primary thread flag the workers to indicate they are now waiting for
     // next parallel region, Also wake them up so they switch their timers to
     // idle.
     for (int i = 0; i < team->t.t_nproc; ++i) {
       kmp_info_t *team_thread = team->t.t_threads[i];
       if (team_thread == this_thr)
         continue;
       team_thread->th.th_stats->setIdleFlag();
       if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME &&
if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) &&
                                loc, nproc);
         break;
       case 2:
         __kmp_itt_frame_submit(gtid, this_thr->th.th_bar_min_time, cur_time, 1,
                                loc, nproc);
         break;
       case 3:
         if (__itt_metadata_add_ptr) {
-          // Initialize with master's wait time
+          // Initialize with primary thread's wait time
           kmp_uint64 delta = cur_time - this_thr->th.th_bar_arrive_time;
           // Set arrive time to zero to be able to check it in
           // __kmp_invoke_task(); the same is done inside the loop below
           this_thr->th.th_bar_arrive_time = 0;
           for (i = 1; i < nproc; ++i) {
             delta += (cur_time - other_threads[i]->th.th_bar_arrive_time);
             other_threads[i]->th.th_bar_arrive_time = 0;
           }
#if USE_ITT_BUILD
   void *itt_sync_obj = NULL;
 #endif /* USE_ITT_BUILD */
   if (team)
     ANNOTATE_BARRIER_END(&team->t.t_bar);
 
   KA_TRACE(10, ("__kmp_fork_barrier: T#%d(%d:%d) has arrived\n", gtid,
                 (team != NULL) ? team->t.t_id : -1, tid));
 
-  // th_team pointer only valid for master thread here
+  // th_team pointer only valid for primary thread here
   if (KMP_MASTER_TID(tid)) {
 #if USE_ITT_BUILD && USE_ITT_NOTIFY
     if (__itt_sync_create_ptr || KMP_ITT_DEBUG) {
       // Create itt barrier object
       itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier, 1);
       __kmp_itt_barrier_middle(gtid, itt_sync_obj); // Call acquired/releasing
     }
 #endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */
#ifdef KMP_DEBUG
     }
 #endif
 
     if (__kmp_tasking_mode != tskm_immediate_exec) {
       // 0 indicates setup current task team if nthreads > 1
       __kmp_task_team_setup(this_thr, team, 0);
     }
 
-    /* The master thread may have changed its blocktime between the join barrier
-       and the fork barrier. Copy the blocktime info to the thread, where
+    /* The primary thread may have changed its blocktime between join barrier
+       and fork barrier. Copy the blocktime info to the thread, where
        __kmp_wait_template() can access it when the team struct is not
        guaranteed to exist. */
     // See note about the corresponding code in __kmp_join_barrier() being
     // performance-critical
     if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
 #if KMP_USE_MONITOR
       this_thr->th.th_team_bt_intervals =
           team->t.t_implicit_task_taskdata[tid].td_icvs.bt_intervals;
       this_thr->th.th_team_bt_set =
           team->t.t_implicit_task_taskdata[tid].td_icvs.bt_set;
 #else
       this_thr->th.th_team_bt_intervals = KMP_BLOCKTIME_INTERVAL(team, tid);
 #endif
     }
-  } // master
+  } // primary thread
 
   switch (__kmp_barrier_release_pattern[bs_forkjoin_barrier]) {
   case bp_hyper_bar: {
     KMP_ASSERT(__kmp_barrier_release_branch_bits[bs_forkjoin_barrier]);
     __kmp_hyper_barrier_release(bs_forkjoin_barrier, this_thr, gtid, tid,
                                 TRUE USE_ITT_BUILD_ARG(itt_sync_obj));
     break;
   }
if (__itt_sync_create_ptr || KMP_ITT_DEBUG) {
       }
     }
 #endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */
     KA_TRACE(10, ("__kmp_fork_barrier: T#%d is leaving early\n", gtid));
     return;
   }
 
   /* We can now assume that a valid team structure has been allocated by the
-     master and propagated to all worker threads. The current thread, however,
-     may not be part of the team, so we can't blindly assume that the team
-     pointer is non-null.  */
+     primary thread and propagated to all worker threads. The current thread,
+     however, may not be part of the team, so we can't blindly assume that the
+     team pointer is non-null.  */
   team = (kmp_team_t *)TCR_PTR(this_thr->th.th_team);
   KMP_DEBUG_ASSERT(team != NULL);
   tid = __kmp_tid_from_gtid(gtid);
 
 #if KMP_BARRIER_ICV_PULL
-  /* Master thread's copy of the ICVs was set up on the implicit taskdata in
-     __kmp_reinitialize_team. __kmp_fork_call() assumes the master thread's
+  /* Primary thread's copy of the ICVs was set up on the implicit taskdata in
+     __kmp_reinitialize_team. __kmp_fork_call() assumes the primary thread's
      implicit task has this data before this function is called. We cannot
-     modify __kmp_fork_call() to look at the fixed ICVs in the master's thread
-     struct, because it is not always the case that the threads arrays have
-     been allocated when __kmp_fork_call() is executed. */
+     modify __kmp_fork_call() to look at the fixed ICVs in the primary thread's
+     thread struct, because it is not always the case that the threads arrays
+     have been allocated when __kmp_fork_call() is executed. */
   {
     KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_icv_copy);
-    if (!KMP_MASTER_TID(tid)) { // master thread already has ICVs
-      // Copy the initial ICVs from the master's thread struct to the implicit
-      // task for this tid.
+    if (!KMP_MASTER_TID(tid)) { // primary thread already has ICVs
+      // Copy the initial ICVs from the primary thread's thread struct to the
+      // implicit task for this tid.
       KA_TRACE(10,
                ("__kmp_fork_barrier: T#%d(%d) is PULLing ICVs\n", gtid, tid));
       __kmp_init_implicit_task(team->t.t_ident, team->t.t_threads[tid], team,
                                tid, FALSE);
       copy_icvs(&team->t.t_implicit_task_taskdata[tid].td_icvs,
                 &team->t.t_threads[0]
                      ->th.th_bar[bs_forkjoin_barrier]
                      .bb.th_fixed_icvs);

 
 void __kmp_setup_icv_copy(kmp_team_t *team, int new_nproc,
                           kmp_internal_control_t *new_icvs, ident_t *loc) {
   KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_setup_icv_copy);
 
   KMP_DEBUG_ASSERT(team && new_nproc && new_icvs);
   KMP_DEBUG_ASSERT((!TCR_4(__kmp_init_parallel)) || new_icvs->nproc);
 
-/* Master thread's copy of the ICVs was set up on the implicit taskdata in
-   __kmp_reinitialize_team. __kmp_fork_call() assumes the master thread's
+/* Primary thread's copy of the ICVs was set up on the implicit taskdata in
+   __kmp_reinitialize_team. __kmp_fork_call() assumes the primary thread's
    implicit task has this data before this function is called. */
 #if KMP_BARRIER_ICV_PULL
-  /* Copy ICVs to master's thread structure into th_fixed_icvs (which remains
-     untouched), where all of the worker threads can access them and make their
-     own copies after the barrier. */
+  /* Copy ICVs to primary thread's thread structure into th_fixed_icvs (which
+     remains untouched), where all of the worker threads can access them and
+     make their own copies after the barrier. */
   KMP_DEBUG_ASSERT(team->t.t_threads[0]); // The threads arrays should be
   // allocated at this point
   copy_icvs(
       &team->t.t_threads[0]->th.th_bar[bs_forkjoin_barrier].bb.th_fixed_icvs,
       new_icvs);
   KF_TRACE(10, ("__kmp_setup_icv_copy: PULL: T#%d this_thread=%p team=%p\n", 0,
                 team->t.t_threads[0], team));
 #elif KMP_BARRIER_ICV_PUSH
   // The ICVs will be propagated in the fork barrier, so nothing needs to be
   // done here.
   KF_TRACE(10, ("__kmp_setup_icv_copy: PUSH: T#%d this_thread=%p team=%p\n", 0,
                 team->t.t_threads[0], team));
 #else
-  // Copy the ICVs to each of the non-master threads.  This takes O(nthreads)
+  // Copy the ICVs to each of the non-primary threads.  This takes O(nthreads)
   // time.
   ngo_load(new_icvs);
   KMP_DEBUG_ASSERT(team->t.t_threads[0]); // The threads arrays should be
   // allocated at this point
-  for (int f = 1; f < new_nproc; ++f) { // Skip the master thread
+  for (int f = 1; f < new_nproc; ++f) { // Skip the primary thread
     // TODO: GEH - pass in better source location info since usually NULL here
     KF_TRACE(10, ("__kmp_setup_icv_copy: LINEAR: T#%d this_thread=%p team=%p\n",
                   f, team->t.t_threads[f], team));
     __kmp_init_implicit_task(loc, team->t.t_threads[f], team, f, FALSE);
     ngo_store_icvs(&team->t.t_implicit_task_taskdata[f].td_icvs, new_icvs);
     KF_TRACE(10, ("__kmp_setup_icv_copy: LINEAR: T#%d this_thread=%p team=%p\n",
                   f, team->t.t_threads[f], team));
   }
   ngo_sync();
 #endif // KMP_BARRIER_ICV_PULL
 }

openmp/runtime/src/kmp_csupport.cpp

 @return The global thread index of the active thread.
 
 This function can be called in any context.
 
 If the runtime has ony been entered at the outermost level from a
 single (necessarily non-OpenMP<sup>*</sup>) thread, then the thread number is
 that which would be returned by omp_get_thread_num() in the outermost
 active parallel construct. (Or zero if there is no active parallel
-construct, since the master thread is necessarily thread zero).
+construct, since the primary thread is necessarily thread zero).
 
 If multiple non-OpenMP threads all enter an OpenMP construct then this
 will be a unique thread identifier among all the threads created by
 the OpenMP runtime (but the value cannot be defined in terms of
 OpenMP thread ids returned by omp_get_thread_num()).
 */
 kmp_int32 __kmpc_global_thread_num(ident_t *loc) {
   kmp_int32 gtid = __kmp_entry_gtid();
if (__kmp_env_consistency_check) {
     if (KMP_MASTER_GTID(global_tid))
       __kmp_pop_sync(global_tid, ct_master, loc);
   }
 }
 
 /*!
 @ingroup WORK_SHARING
 @param loc  source location information.
+@param global_tid  global thread number.
+@param filter result of evaluating filter clause on thread global_tid, or zero
+if no filter clause present
+@return 1 if this thread should execute the <tt>masked</tt> block, 0 otherwise.
+*/
+kmp_int32 __kmpc_masked(ident_t *loc, kmp_int32 global_tid, kmp_int32 filter) {
+  int status = 0;
+  int tid;
+  KC_TRACE(10, ("__kmpc_masked: called T#%d\n", global_tid));
+  __kmp_assert_valid_gtid(global_tid);
+
+  if (!TCR_4(__kmp_init_parallel))
+    __kmp_parallel_initialize();
+
+  __kmp_resume_if_soft_paused();
+
+  tid = __kmp_tid_from_gtid(global_tid);
+  if (tid == filter) {
+    KMP_COUNT_BLOCK(OMP_MASKED);
+    KMP_PUSH_PARTITIONED_TIMER(OMP_masked);
+    status = 1;
+  }
+
+#if OMPT_SUPPORT && OMPT_OPTIONAL
+  if (status) {
+    if (ompt_enabled.ompt_callback_masked) {
+      kmp_info_t *this_thr = __kmp_threads[global_tid];
+      kmp_team_t *team = this_thr->th.th_team;
+      ompt_callbacks.ompt_callback(ompt_callback_masked)(
+          ompt_scope_begin, &(team->t.ompt_team_info.parallel_data),
+          &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
+          OMPT_GET_RETURN_ADDRESS(0));
+    }
+  }
+#endif
+
+  if (__kmp_env_consistency_check) {
+#if KMP_USE_DYNAMIC_LOCK
+    if (status)
+      __kmp_push_sync(global_tid, ct_masked, loc, NULL, 0);
+    else
+      __kmp_check_sync(global_tid, ct_masked, loc, NULL, 0);
+#else
+    if (status)
+      __kmp_push_sync(global_tid, ct_masked, loc, NULL);
+    else
+      __kmp_check_sync(global_tid, ct_masked, loc, NULL);
+#endif
+  }
+
+  return status;
+}
+
+/*!
+@ingroup WORK_SHARING
+@param loc  source location information.
+@param global_tid  global thread number .
+
+Mark the end of a <tt>masked</tt> region. This should only be called by the
+thread that executes the <tt>masked</tt> region.
+*/
+void __kmpc_end_masked(ident_t *loc, kmp_int32 global_tid) {
+  KC_TRACE(10, ("__kmpc_end_masked: called T#%d\n", global_tid));
+  __kmp_assert_valid_gtid(global_tid);
+  KMP_POP_PARTITIONED_TIMER();
+
+#if OMPT_SUPPORT && OMPT_OPTIONAL
+  kmp_info_t *this_thr = __kmp_threads[global_tid];
+  kmp_team_t *team = this_thr->th.th_team;
+  if (ompt_enabled.ompt_callback_masked) {
+    int tid = __kmp_tid_from_gtid(global_tid);
+    ompt_callbacks.ompt_callback(ompt_callback_masked)(
+        ompt_scope_end, &(team->t.ompt_team_info.parallel_data),
+        &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
+        OMPT_GET_RETURN_ADDRESS(0));
+  }
+#endif
+
+  if (__kmp_env_consistency_check) {
+    __kmp_pop_sync(global_tid, ct_masked, loc);
+  }
+}
+
+/*!
+@ingroup WORK_SHARING
+@param loc  source location information.
 @param gtid  global thread number.
 
 Start execution of an <tt>ordered</tt> construct.
 */
 void __kmpc_ordered(ident_t *loc, kmp_int32 gtid) {
   int cid = 0;
   kmp_info_t *th;
   KMP_DEBUG_ASSERT(__kmp_init_serial);

 @param loc source location information
 @param global_tid global thread number
 @param num_vars number of items (variables) to be reduced
 @param reduce_size size of data in bytes to be reduced
 @param reduce_data pointer to data to be reduced
 @param reduce_func callback function providing reduction operation on two
 operands and returning result of reduction in lhs_data
 @param lck pointer to the unique lock data structure
-@result 1 for the master thread, 0 for all other team threads, 2 for all team
+@result 1 for the primary thread, 0 for all other team threads, 2 for all team
 threads if atomic reduction needed
 
 The nowait version is used for a reduce clause with the nowait argument.
 */
 kmp_int32
 __kmpc_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars,
                      size_t reduce_size, void *reduce_data,
                      void (*reduce_func)(void *lhs_data, void *rhs_data),
if (packed_reduction_method == critical_reduce_block) {
     //      slightly later, literally on next instruction)
     if (__kmp_env_consistency_check)
       __kmp_pop_sync(global_tid, ct_reduce, loc);
 
   } else if (TEST_REDUCTION_METHOD(packed_reduction_method,
                                    tree_reduce_block)) {
 
 // AT: performance issue: a real barrier here
-// AT:     (if master goes slow, other threads are blocked here waiting for the
-// master to come and release them)
-// AT:     (it's not what a customer might expect specifying NOWAIT clause)
-// AT:     (specifying NOWAIT won't result in improvement of performance, it'll
-// be confusing to a customer)
+// AT: (if primary thread is slow, other threads are blocked here waiting for
+//      the primary thread to come and release them)
+// AT: (it's not what a customer might expect specifying NOWAIT clause)
+// AT: (specifying NOWAIT won't result in improvement of performance, it'll
+//      be confusing to a customer)
 // AT: another implementation of *barrier_gather*nowait() (or some other design)
 // might go faster and be more in line with sense of NOWAIT
 // AT: TO DO: do epcc test and compare times
 
 // this barrier should be invisible to a customer and to the threading profile
 // tool (it's neither a terminating barrier nor customer's code, it's
 // used for an internal purpose)
 #if OMPT_SUPPORT
retval =
                       global_tid, FALSE, reduce_size, reduce_data, reduce_func);
     retval = (retval != 0) ? (0) : (1);
 #if OMPT_SUPPORT && OMPT_OPTIONAL
     if (ompt_enabled.enabled) {
       ompt_frame->enter_frame = ompt_data_none;
     }
 #endif
 
-    // all other workers except master should do this pop here
+    // all other workers except primary thread should do this pop here
     //     ( none of other workers will get to __kmpc_end_reduce_nowait() )
     if (__kmp_env_consistency_check) {
       if (retval == 0) {
         __kmp_pop_sync(global_tid, ct_reduce, loc);
       }
     }
 
   } else {
if (packed_reduction_method == critical_reduce_block) {
 
     // usage: if team size == 1, no synchronization is required ( on Intel
     // platforms only )
 
     OMPT_REDUCTION_END;
 
   } else if (packed_reduction_method == atomic_reduce_block) {
 
-    // neither master nor other workers should get here
+    // neither primary thread nor other workers should get here
     //     (code gen does not generate this call in case 2: atomic reduce block)
     // actually it's better to remove this elseif at all;
     // after removal this value will checked by the 'else' and will assert
 
   } else if (TEST_REDUCTION_METHOD(packed_reduction_method,
                                    tree_reduce_block)) {
 
-    // only master gets here
+    // only primary thread gets here
     // OMPT: tree reduction is annotated in the barrier code
 
   } else {
 
     // should never reach this block
     KMP_ASSERT(0); // "unexpected method"
   }
 

 @param loc source location information
 @param global_tid global thread number
 @param num_vars number of items (variables) to be reduced
 @param reduce_size size of data in bytes to be reduced
 @param reduce_data pointer to data to be reduced
 @param reduce_func callback function providing reduction operation on two
 operands and returning result of reduction in lhs_data
 @param lck pointer to the unique lock data structure
-@result 1 for the master thread, 0 for all other team threads, 2 for all team
+@result 1 for the primary thread, 0 for all other team threads, 2 for all team
 threads if atomic reduction needed
 
 A blocking reduce that includes an implicit barrier.
 */
 kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars,
                         size_t reduce_size, void *reduce_data,
                         void (*reduce_func)(void *lhs_data, void *rhs_data),
                         kmp_critical_name *lck) {
retval =
                       global_tid, TRUE, reduce_size, reduce_data, reduce_func);
     retval = (retval != 0) ? (0) : (1);
 #if OMPT_SUPPORT && OMPT_OPTIONAL
     if (ompt_enabled.enabled) {
       ompt_frame->enter_frame = ompt_data_none;
     }
 #endif
 
-    // all other workers except master should do this pop here
-    // ( none of other workers except master will enter __kmpc_end_reduce() )
+    // all other workers except primary thread should do this pop here
+    // (none of other workers except primary will enter __kmpc_end_reduce())
     if (__kmp_env_consistency_check) {
-      if (retval == 0) { // 0: all other workers; 1: master
+      if (retval == 0) { // 0: all other workers; 1: primary thread
         __kmp_pop_sync(global_tid, ct_reduce, loc);
       }
     }
 
   } else {
 
     // should never reach this block
     KMP_ASSERT(0); // "unexpected method"
#if OMPT_SUPPORT && OMPT_OPTIONAL
     if (ompt_enabled.enabled) {
       ompt_frame->enter_frame = ompt_data_none;
     }
 #endif
 
   } else if (TEST_REDUCTION_METHOD(packed_reduction_method,
                                    tree_reduce_block)) {
 
-    // only master executes here (master releases all other workers)
+    // only primary thread executes here (primary releases all other workers)
     __kmp_end_split_barrier(UNPACK_REDUCTION_BARRIER(packed_reduction_method),
                             global_tid);
 
   } else {
 
     // should never reach this block
     KMP_ASSERT(0); // "unexpected method"
   }

openmp/runtime/src/kmp_dispatch.cpp

if (active) {
     th->th.th_dispatch->th_dispatch_sh_current =
         CCAST(dispatch_shared_info_t *, (volatile dispatch_shared_info_t *)sh);
 #if USE_ITT_BUILD
     if (pr->flags.ordered) {
       __kmp_itt_ordered_init(gtid);
     }
     // Report loop metadata
     if (itt_need_metadata_reporting) {
-      // Only report metadata by master of active team at level 1
+      // Only report metadata by primary thread of active team at level 1
       kmp_uint64 schedtype = 0;
       switch (schedule) {
       case kmp_sch_static_chunked:
       case kmp_sch_static_balanced: // Chunk is calculated in the switch above
         break;
       case kmp_sch_static_greedy:
         cur_chunk = pr->u.p.parm1;
         break;

openmp/runtime/src/kmp_dispatch_hier.h

int next_recurse(ident_t *loc, int gtid, kmp_hier_top_unit_t<T> *current,
     KMP_DEBUG_ASSERT(parent || last_layer);
 
     KD_TRACE(
         1, ("kmp_hier_t.next_recurse(): T#%d (%d) called\n", gtid, hier_level));
 
     T hier_id = (T)current->get_hier_id();
     // Attempt to grab next iteration range for this level
     if (previous_id == 0) {
-      KD_TRACE(1, ("kmp_hier_t.next_recurse(): T#%d (%d) is master of unit\n",
+      KD_TRACE(1, ("kmp_hier_t.next_recurse(): T#%d (%d) is primary of unit\n",
                    gtid, hier_level));
       kmp_int32 contains_last;
       T my_lb, my_ub;
       ST my_st;
       T nproc;
       dispatch_shared_info_template<T> volatile *my_sh;
       dispatch_private_info_template<T> *my_pr;
       if (last_layer) {
#endif
                         gtid, hier_level));
           current->hier_pr.flags.contains_last = contains_last;
         }
         if (!current->hier_pr.flags.contains_last)
           contains_last = FALSE;
       }
       if (p_last)
         *p_last = contains_last;
-    } // if master thread of this unit
+    } // if primary thread of this unit
     if (hier_level > 0 || !__kmp_dispatch_hand_threading) {
       KD_TRACE(10,
                ("kmp_hier_t.next_recurse(): T#%d (%d) going into barrier.\n",
                 gtid, hier_level));
       current->barrier(previous_id, tdata);
       KD_TRACE(10,
                ("kmp_hier_t.next_recurse(): T#%d (%d) released and exit %d\n",
                 gtid, hier_level, current->get_curr_status(tdata->index)));
int next(ident_t *loc, int gtid, dispatch_private_info_template<T> *pr,
     //       threads 4,5,6,7 all execute iteration 3
     //       ... etc.
     if (__kmp_dispatch_hand_threading) {
       KD_TRACE(10,
                ("kmp_hier_t.next(): T#%d THREAD LEVEL using hand threading\n",
                 gtid));
       if (unit_id == 0) {
         // For hand threading, the sh buffer on the lowest level is only ever
-        // modified and read by the master thread on that level.  Because of
+        // modified and read by the primary thread on that level.  Because of
         // this, we can always use the first sh buffer.
         auto sh = &(parent->hier_barrier.sh[0]);
         KMP_DEBUG_ASSERT(sh);
         status = __kmp_dispatch_next_algorithm<T>(
             gtid, pr, sh, &contains_last, p_lb, p_ub, p_st, nproc, unit_id);
         if (!status) {
           bool done = false;
           while (!done) {
#endif
               KD_TRACE(10, ("kmp_hier_t.next(): T#%d THREAD LEVEL status == 2 "
                             "trying again.\n",
                             gtid));
               done = false;
             }
           }
         }
         parent->set_next_hand_thread(*p_lb, *p_ub, *p_st, status, tdata->index);
-      } // if master thread of lowest unit level
+      } // if primary thread of lowest unit level
       parent->barrier(pr->get_hier_id(), tdata);
       if (unit_id != 0) {
         *p_lb = parent->get_curr_lb(tdata->index);
         *p_ub = parent->get_curr_ub(tdata->index);
         *p_st = parent->get_curr_st(tdata->index);
         status = parent->get_curr_status(tdata->index);
       }
     } else {
if (!active) {
     pr->flags.use_hier = FALSE;
     pr->flags.contains_last = FALSE;
     return;
   }
   KMP_DEBUG_ASSERT(pr);
   KMP_DEBUG_ASSERT(sh);
   pr->flags.use_hier = TRUE;
   pr->u.p.tc = 0;
-  // Have master allocate the hierarchy
+  // Have primary thread allocate the hierarchy
   if (__kmp_tid_from_gtid(gtid) == 0) {
     KD_TRACE(10, ("__kmp_dispatch_init_hierarchy: T#%d pr:%p sh:%p allocating "
                   "hierarchy\n",
                   gtid, pr, sh));
     if (sh->hier == NULL) {
       sh->hier = (kmp_hier_t<T> *)__kmp_allocate(sizeof(kmp_hier_t<T>));
     }
     sh->hier->allocate_hier(n, new_layers, new_scheds, new_chunks);
#endif // KMP_DEBUG
   // the barrier data for each unit can be initialized and the last layer's
   // loop information can be initialized.
   int prev_id = pr->get_hier_id();
   for (int i = 0; i < n; ++i) {
     if (prev_id != 0)
       break;
     int index = __kmp_dispatch_get_index(tid, hier->get_type(i));
     kmp_hier_top_unit_t<T> *my_unit = hier->get_unit(i, index);
-    // Only master threads of this unit within the hierarchy do initialization
+    // Only primary threads of this unit within the hierarchy do initialization
     KD_TRACE(10, ("__kmp_dispatch_init_hierarchy: T#%d (%d) prev_id is 0\n",
                   gtid, i));
     my_unit->reset_shared_barrier();
     my_unit->hier_pr.flags.contains_last = FALSE;
     // Last layer, initialize the private buffers with entire loop information
     // Now the next next_algorithm() call will get the first chunk of
     // iterations properly
     if (i == n - 1) {

openmp/runtime/src/kmp_error.cpp

"(none)", "\"parallel\"", "work-sharing", /* this is not called "for"
                                                  "sections" pragmas */
     "\"ordered\" work-sharing", /* this is not called "for ordered" because of
                                    lowering of "sections" pragmas */
     "\"sections\"",
     "work-sharing", /* this is not called "single" because of lowering of
                        "sections" pragmas */
     "\"critical\"", "\"ordered\"", /* in PARALLEL */
     "\"ordered\"", /* in PDO */
-    "\"master\"", "\"reduce\"", "\"barrier\""};
+    "\"master\"", "\"reduce\"", "\"barrier\"", "\"masked\""};
 
 #define get_src(ident) ((ident) == NULL ? NULL : (ident)->psource)
 
 #define PUSH_MSG(ct, ident)                                                    \
   "\tpushing on stack: %s (%s)\n", cons_text_c[(ct)], get_src((ident))
 #define POP_MSG(p)                                                             \
   "\tpopping off stack: %s (%s)\n", cons_text_c[(p)->stack_data[tos].type],    \
       get_src((p)->stack_data[tos].ident)
#endif
       if (index != 0) {
         /* found match on the stack (may not always because of interleaved
          * critical for Fortran) */
         cons = p->stack_data[index];
       }
       /* we are in CRITICAL which is inside a CRITICAL construct of same name */
       __kmp_error_construct2(kmp_i18n_msg_CnsNestingSameName, ct, ident, &cons);
     }
-  } else if (ct == ct_master || ct == ct_reduce) {
+  } else if (ct == ct_master || ct == ct_masked || ct == ct_reduce) {
     if (p->w_top > p->p_top) {
       /* inside a WORKSHARING construct for this PARALLEL region */
       __kmp_error_construct2(kmp_i18n_msg_CnsInvalidNesting, ct, ident,
                              &p->stack_data[p->w_top]);
     }
     if (ct == ct_reduce && p->s_top > p->p_top) {
       /* inside a another SYNC construct for this PARALLEL region */
       __kmp_error_construct2(kmp_i18n_msg_CnsInvalidNesting, ct, ident,

openmp/runtime/src/kmp_global.cpp

 // This will be 1 if KMP_USE_YIELD environment variable was set explicitly
 kmp_int32 __kmp_use_yield_exp_set = 0;
 
 kmp_uint32 __kmp_yield_init = KMP_INIT_WAIT;
 kmp_uint32 __kmp_yield_next = KMP_NEXT_WAIT;
 
 /* ------------------------------------------------------ */
 /* STATE mostly syncronized with global lock */
-/* data written to rarely by masters, read often by workers */
+/* data written to rarely by primary threads, read often by workers */
 /* TODO: None of this global padding stuff works consistently because the order
    of declaration is not necessarily correlated to storage order. To fix this,
    all the important globals must be put in a big structure instead. */
 KMP_ALIGN_CACHE
 kmp_info_t **__kmp_threads = NULL;
 kmp_root_t **__kmp_root = NULL;
 
-/* data read/written to often by masters */
+/* data read/written to often by primary threads */
 KMP_ALIGN_CACHE
 volatile int __kmp_nth = 0;
 volatile int __kmp_all_nth = 0;
 volatile kmp_info_t *__kmp_thread_pool = NULL;
 volatile kmp_team_t *__kmp_team_pool = NULL;
 
 KMP_ALIGN_CACHE
 std::atomic<int> __kmp_thread_pool_active_nth = ATOMIC_VAR_INIT(0);

openmp/runtime/src/kmp_gsupport.cpp

 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_DOACROSS_ULL_WAIT)(
     unsigned long long first, ...) {
   va_list args;
   va_start(args, first);
   __kmp_GOMP_doacross_wait<unsigned long long>(first, args);
   va_end(args);
 }
 
-// fn: the function each master thread of new team will call
+// fn: the function each primary thread of new team will call
 // data: argument to fn
 // num_teams, thread_limit: max bounds on respective ICV
 // flags: unused
 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TEAMS_REG)(void (*fn)(void *),
                                                   void *data,
                                                   unsigned num_teams,
                                                   unsigned thread_limit,
                                                   unsigned flags) {

openmp/runtime/src/kmp_itt.h

 
 // Note the naming convention:
 //     __kmp_itt_xxxing() function should be called before action, while
 //     __kmp_itt_xxxed()  function should be called after action.
 
 // --- Parallel region reporting ---
 __kmp_inline void
 __kmp_itt_region_forking(int gtid, int team_size,
-                         int barriers); // Master only, before forking threads.
+                         int barriers); // Primary only, before forking threads.
 __kmp_inline void
-__kmp_itt_region_joined(int gtid); // Master only, after joining threads.
+__kmp_itt_region_joined(int gtid); // Primary only, after joining threads.
 // (*) Note: A thread may execute tasks after this point, though.
 
 // --- Frame reporting ---
 // region=0: no regions, region=1: parallel, region=2: serialized parallel
 __kmp_inline void __kmp_itt_frame_submit(int gtid, __itt_timestamp begin,
                                          __itt_timestamp end, int imbalance,
                                          ident_t *loc, int team_size,
                                          int region = 0);

 /* Markers for the start and end of regions that represent polling and are
    therefore uninteresting to architectural simulations 0x4376 and 0x4377 are
    arbitrary numbers that should be unique in the space of SSC tags, but there
    is no central issuing authority rather randomness is expected to work. */
 #define SSC_MARK_SPIN_START() INSERT_SSC_MARK(0x4376)
 #define SSC_MARK_SPIN_END() INSERT_SSC_MARK(0x4377)
 
 // Markers for architecture simulation.
-// FORKING      : Before the master thread forks.
+// FORKING      : Before the primary thread forks.
 // JOINING      : At the start of the join.
 // INVOKING     : Before the threads invoke microtasks.
 // DISPATCH_INIT: At the start of dynamically scheduled loop.
 // DISPATCH_NEXT: After claming next iteration of dynamically scheduled loop.
 #define SSC_MARK_FORKING() INSERT_SSC_MARK(0xd693)
 #define SSC_MARK_JOINING() INSERT_SSC_MARK(0xd694)
 #define SSC_MARK_INVOKING() INSERT_SSC_MARK(0xd695)
 #define SSC_MARK_DISPATCH_INIT() INSERT_SSC_MARK(0xd696)

openmp/runtime/src/kmp_itt.inl

 #endif
 #endif
 #endif
 
 static kmp_bootstrap_lock_t metadata_lock =
     KMP_BOOTSTRAP_LOCK_INITIALIZER(metadata_lock);
 
 /* Parallel region reporting.
- * __kmp_itt_region_forking should be called by master thread of a team.
+ * __kmp_itt_region_forking should be called by primary thread of a team.
    Exact moment of call does not matter, but it should be completed before any
    thread of this team calls __kmp_itt_region_starting.
  * __kmp_itt_region_starting should be called by each thread of a team just
    before entering parallel region body.
  * __kmp_itt_region_finished should be called by each thread of a team right
    after returning from parallel region body.
- * __kmp_itt_region_joined should be called by master thread of a team, after
+ * __kmp_itt_region_joined should be called by primary thread of a team, after
    all threads called __kmp_itt_region_finished.
 
  Note: Thread waiting at join barrier (after __kmp_itt_region_finished) can
  execute some more user code -- such a thread can execute tasks.
 
  Note: The overhead of logging region_starting and region_finished in each
  thread is too large, so these calls are not used. */
 
if (loc && loc->reserved_2) {
     }
   }
 #endif
 } // __kmp_itt_region_joined
 
 /* Barriers reporting.
 
    A barrier consists of two phases:
-   1. Gather -- master waits for arriving of all the worker threads; each
+   1. Gather -- primary thread waits for all worker threads to arrive; each
       worker thread registers arrival and goes further.
-   2. Release -- each worker threads waits until master lets it go; master lets
-      worker threads go.
+   2. Release -- each worker thread waits until primary thread lets it go;
+      primary thread lets worker threads go.
 
    Function should be called by each thread:
    * __kmp_itt_barrier_starting() -- before arriving to the gather phase.
    * __kmp_itt_barrier_middle()   -- between gather and release phases.
    * __kmp_itt_barrier_finished() -- after release phase.
 
    Note: Call __kmp_itt_barrier_object() before call to
    __kmp_itt_barrier_starting() and save result in local variable.
#if USE_ITT_NOTIFY
   kmp_info_t *thr = __kmp_thread_from_gtid(gtid);
   kmp_team_t *team = thr->th.th_team;
 
   // NOTE: If the function is called from __kmp_fork_barrier, team pointer can
   // be NULL. This "if" helps to avoid crash. However, this is not complete
   // solution, and reporting fork/join barriers to ITT should be revisited.
 
   if (team != NULL) {
-    // Master thread increases b_arrived by KMP_BARRIER_STATE_BUMP each time.
+    // Primary thread increases b_arrived by KMP_BARRIER_STATE_BUMP each time.
     // Divide b_arrived by KMP_BARRIER_STATE_BUMP to get plain barrier counter.
     kmp_uint64 counter =
         team->t.t_bar[bt].b_arrived / KMP_BARRIER_STATE_BUMP + delta;
     // Now form the barrier id. Encode barrier type (bt) in barrier id too, so
     // barriers of different types do not have the same ids.
     KMP_BUILD_ASSERT(sizeof(kmp_team_t) >= bs_last_barrier);
     // This condition is a must (we would have zero divide otherwise).
     KMP_BUILD_ASSERT(sizeof(kmp_team_t) >= 2 * bs_last_barrier);
if (set_name) {
           }
         } else if (expl) {
           type = "OMP Explicit Barrier";
         }
       } break;
       case bs_forkjoin_barrier: {
         // In case of fork/join barrier we can read thr->th.th_ident, because it
         // contains location of last passed construct (while join barrier is not
-        // such one). Use th_ident of master thread instead -- __kmp_join_call()
-        // called by the master thread saves location.
+        // such one). Use th_ident of primary thread instead --
+        // __kmp_join_call() called by the primary thread saves location.
         //
-        // AC: cannot read from master because __kmp_join_call may be not called
-        //    yet, so we read the location from team. This is the same location.
-        //    And team is valid at the enter to join barrier where this happens.
+        // AC: cannot read from primary thread because __kmp_join_call may not
+        //    be called yet, so we read the location from team. This is the
+        //    same location. Team is valid on entry to join barrier where this
+        //    happens.
         loc = team->t.t_ident;
         if (loc != NULL) {
           src = loc->psource;
         }
         type = "OMP Join Barrier";
       } break;
       }
       KMP_ITT_DEBUG_LOCK();

 } // __kmp_itt_thread_ignore
 
 void __kmp_itt_thread_name(int gtid) {
 #if USE_ITT_NOTIFY
   if (__itt_thr_name_set_ptr) {
     kmp_str_buf_t name;
     __kmp_str_buf_init(&name);
     if (KMP_MASTER_GTID(gtid)) {
-      __kmp_str_buf_print(&name, "OMP Master Thread #%d", gtid);
+      __kmp_str_buf_print(&name, "OMP Primary Thread #%d", gtid);
     } else {
       __kmp_str_buf_print(&name, "OMP Worker Thread #%d", gtid);
     }
     KMP_ITT_DEBUG_LOCK();
     __itt_thr_name_set(name.str, name.used);
     KMP_ITT_DEBUG_PRINT("[thr nam] name( \"%s\")\n", name.str);
     __kmp_str_buf_free(&name);
   }
#if USE_ITT_NOTIFY
   KMP_ITT_DEBUG_LOCK();
   __itt_sync_create(object, "OMP Scheduler", name, 0);
   KMP_ITT_DEBUG_PRINT("[sys obj] scre( %p, \"OMP Scheduler\", \"%s\", 0 )\n",
                       object, name);
 #endif
 } // __kmp_itt_system_object_created
 
 /* Stack stitching api.
-   Master calls "create" and put the stitching id into team structure.
+   Primary thread calls "create" and put the stitching id into team structure.
    Workers read the stitching id and call "enter" / "leave" api.
-   Master calls "destroy" at the end of the parallel region. */
+   Primary thread calls "destroy" at the end of the parallel region. */
 
 __itt_caller __kmp_itt_stack_caller_create() {
 #if USE_ITT_NOTIFY
   if (!__itt_stack_caller_create_ptr)
     return NULL;
   KMP_ITT_DEBUG_LOCK();
   __itt_caller id = __itt_stack_caller_create();
   KMP_ITT_DEBUG_PRINT("[stk cre] %p\n", id);

openmp/runtime/src/kmp_omp.h

#endif
   /* kmp_desc structure (for info field above) */
   kmp_int32 ds_sizeof_struct;
   offset_and_size_t ds_tid; // team thread id
   offset_and_size_t ds_gtid; // global thread id
   offset_and_size_t ds_thread; // native thread id
 
   /* team structure information */
   kmp_int32 t_sizeof_struct;
-  offset_and_size_t t_master_tid; // tid of master in parent team
+  offset_and_size_t t_master_tid; // tid of primary thread in parent team
   offset_and_size_t t_ident; // location of parallel region
   offset_and_size_t t_parent; // parent team
   offset_and_size_t t_nproc; // # team threads
   offset_and_size_t t_threads; // array of threads
   offset_and_size_t t_serialized; // # levels of serialized teams
   offset_and_size_t t_id; // unique team id
   offset_and_size_t t_pkfn;
   offset_and_size_t t_task_team; // task team structure
   offset_and_size_t t_implicit_task; // taskdata for the thread's implicit task
   offset_and_size_t t_cancel_request;
   offset_and_size_t t_bar;
   offset_and_size_t
-      t_b_master_arrived; // increased by 1 when master arrives to a barrier
+      t_b_master_arrived; // incremented when primary thread reaches barrier
   offset_and_size_t
       t_b_team_arrived; // increased by one when all the threads arrived
 
   /* root structure information */
   kmp_int32 r_sizeof_struct;
   offset_and_size_t r_root_team; // team at root
   offset_and_size_t r_hot_team; // hot team for this root
   offset_and_size_t r_uber_thread; // root thread

openmp/runtime/src/kmp_runtime.cpp

if (team->t.t_serialized) {
     /* TODO: Should this be acquire or release? */
     if (team->t.t_construct == old_this) {
       status = __kmp_atomic_compare_store_acq(&team->t.t_construct, old_this,
                                               th->th.th_local.this_construct);
     }
 #if USE_ITT_BUILD
     if (__itt_metadata_add_ptr && __kmp_forkjoin_frames_mode == 3 &&
         KMP_MASTER_GTID(gtid) && th->th.th_teams_microtask == NULL &&
-        team->t.t_active_level ==
-            1) { // Only report metadata by master of active team at level 1
+        team->t.t_active_level == 1) {
+      // Only report metadata by primary thread of active team at level 1
       __kmp_itt_metadata_single(id_ref);
     }
 #endif /* USE_ITT_BUILD */
   }
 
   if (__kmp_env_consistency_check) {
     if (status && push_ws) {
       __kmp_push_workshare(gtid, ct_psingle, id_ref);
static void __kmp_fork_team_threads(kmp_root_t *root, kmp_team_t *team,
                                     kmp_info_t *master_th, int master_gtid) {
   int i;
   int use_hot_team;
 
   KA_TRACE(10, ("__kmp_fork_team_threads: new_nprocs = %d\n", team->t.t_nproc));
   KMP_DEBUG_ASSERT(master_gtid == __kmp_get_gtid());
   KMP_MB();
 
-  /* first, let's setup the master thread */
+  /* first, let's setup the primary thread */
   master_th->th.th_info.ds.ds_tid = 0;
   master_th->th.th_team = team;
   master_th->th.th_team_nproc = team->t.t_nproc;
   master_th->th.th_team_master = master_th;
   master_th->th.th_team_serialized = FALSE;
   master_th->th.th_dispatch = &team->t.t_dispatch[0];
 
 /* make sure we are not the optimized hot team */
if (level < __kmp_hot_teams_max_level) {
       use_hot_team = 0;
     }
   }
 #else
   use_hot_team = team == root->r.r_hot_team;
 #endif
   if (!use_hot_team) {
 
-    /* install the master thread */
+    /* install the primary thread */
     team->t.t_threads[0] = master_th;
     __kmp_initialize_info(master_th, team, 0, master_gtid);
 
     /* now, install the worker threads */
     for (i = 1; i < team->t.t_nproc; i++) {
 
       /* fork or reallocate a new thread and install it in team */
       kmp_info_t *thr = __kmp_allocate_thread(root, team, i);

 // Propagate any changes to the floating point control registers out to the team
 // We try to avoid unnecessary writes to the relevant cache line in the team
 // structure, so we don't make changes unless they are needed.
 inline static void propagateFPControl(kmp_team_t *team) {
   if (__kmp_inherit_fp_control) {
     kmp_int16 x87_fpu_control_word;
     kmp_uint32 mxcsr;
 
-    // Get master values of FPU control flags (both X87 and vector)
+    // Get primary thread's values of FPU control flags (both X87 and vector)
     __kmp_store_x87_fpu_control_word(&x87_fpu_control_word);
     __kmp_store_mxcsr(&mxcsr);
     mxcsr &= KMP_X86_MXCSR_MASK;
 
     // There is no point looking at t_fp_control_saved here.
     // If it is TRUE, we still have to update the values if they are different
     // from those we now have. If it is FALSE we didn't save anything yet, but
     // our objective is the same. We have to ensure that the values in the team

 #define propagateFPControl(x) ((void)0)
 #define updateHWFPControl(x) ((void)0)
 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
 
 static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team,
                                      int realloc); // forward declaration
 
 /* Run a parallel region that has been serialized, so runs only in a team of the
-   single master thread. */
+   single primary thread. */
 void __kmp_serialized_parallel(ident_t *loc, kmp_int32 global_tid) {
   kmp_info_t *this_thr;
   kmp_team_t *serial_team;
 
   KC_TRACE(10, ("__kmpc_serialized_parallel: called by T#%d\n", global_tid));
 
   /* Skip all this code for autopar serialized loops since it results in
      unacceptable overhead */
#endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */
       __kmp_internal_fork(loc, gtid, parent_team);
       KF_TRACE(10, ("__kmp_fork_call: after internal fork: root=%p, team=%p, "
                     "master_th=%p, gtid=%d\n",
                     root, parent_team, master_th, gtid));
 
       if (call_context == fork_context_gnu)
         return TRUE;
 
-      /* Invoke microtask for MASTER thread */
+      /* Invoke microtask for PRIMARY thread */
       KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid,
                     parent_team->t.t_id, parent_team->t.t_pkfn));
 
       if (!parent_team->t.t_invoke(gtid)) {
-        KMP_ASSERT2(0, "cannot invoke microtask for MASTER thread");
+        KMP_ASSERT2(0, "cannot invoke microtask for PRIMARY thread");
       }
       KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid,
                     parent_team->t.t_id, parent_team->t.t_pkfn));
       KMP_MB(); /* Flush all pending memory write invalidates.  */
 
       KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid));
 
       return TRUE;
#endif
     } else {
       // AC: Do not increase parallel level at start of the teams construct
       int new_level = parent_team->t.t_level;
       KMP_CHECK_UPDATE(team->t.t_level, new_level);
       new_level = parent_team->t.t_active_level;
       KMP_CHECK_UPDATE(team->t.t_active_level, new_level);
     }
     kmp_r_sched_t new_sched = get__sched_2(parent_team, master_tid);
-    // set master's schedule as new run-time schedule
+    // set primary thread's schedule as new run-time schedule
     KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched);
 
     KMP_CHECK_UPDATE(team->t.t_cancel_request, cancel_noreq);
     KMP_CHECK_UPDATE(team->t.t_def_allocator, master_th->th.th_def_allocator);
 
     // Update the floating point rounding in the team if required.
     propagateFPControl(team);
 
     if (__kmp_tasking_mode != tskm_immediate_exec) {
-      // Set master's task team to team's task team. Unless this is hot team, it
-      // should be NULL.
+      // Set primary thread's task team to team's task team. Unless this is hot
+      // team, it should be NULL.
       KMP_DEBUG_ASSERT(master_th->th.th_task_team ==
                        parent_team->t.t_task_team[master_th->th.th_task_state]);
-      KA_TRACE(20, ("__kmp_fork_call: Master T#%d pushing task_team %p / team "
+      KA_TRACE(20, ("__kmp_fork_call: Primary T#%d pushing task_team %p / team "
                     "%p, new task_team %p / team %p\n",
                     __kmp_gtid_from_thread(master_th),
                     master_th->th.th_task_team, parent_team,
                     team->t.t_task_team[master_th->th.th_task_state], team));
 
       if (active_level || master_th->th.th_task_team) {
-        // Take a memo of master's task_state
+        // Take a memo of primary thread's task_state
         KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack);
         if (master_th->th.th_task_state_top >=
             master_th->th.th_task_state_stack_sz) { // increase size
           kmp_uint32 new_size = 2 * master_th->th.th_task_state_stack_sz;
           kmp_uint8 *old_stack, *new_stack;
           kmp_uint32 i;
           new_stack = (kmp_uint8 *)__kmp_allocate(new_size);
           for (i = 0; i < master_th->th.th_task_state_stack_sz; ++i) {
             new_stack[i] = master_th->th.th_task_state_memo_stack[i];
           }
           for (i = master_th->th.th_task_state_stack_sz; i < new_size;
                ++i) { // zero-init rest of stack
             new_stack[i] = 0;
           }
           old_stack = master_th->th.th_task_state_memo_stack;
           master_th->th.th_task_state_memo_stack = new_stack;
           master_th->th.th_task_state_stack_sz = new_size;
           __kmp_free(old_stack);
         }
-        // Store master's task_state on stack
+        // Store primary thread's task_state on stack
         master_th->th
             .th_task_state_memo_stack[master_th->th.th_task_state_top] =
             master_th->th.th_task_state;
         master_th->th.th_task_state_top++;
 #if KMP_NESTED_HOT_TEAMS
         if (master_th->th.th_hot_teams &&
             active_level < __kmp_hot_teams_max_level &&
             team == master_th->th.th_hot_teams[active_level].hot_team) {
-          // Restore master's nested state if nested hot team
+          // Restore primary thread's nested state if nested hot team
           master_th->th.th_task_state =
               master_th->th
                   .th_task_state_memo_stack[master_th->th.th_task_state_top];
         } else {
 #endif
           master_th->th.th_task_state = 0;
 #if KMP_NESTED_HOT_TEAMS
         }

 #if USE_ITT_BUILD
     if (__itt_stack_caller_create_ptr) {
       team->t.t_stack_id =
           __kmp_itt_stack_caller_create(); // create new stack stitching id
       // before entering fork barrier
     }
 #endif /* USE_ITT_BUILD */
 
-    // AC: skip __kmp_internal_fork at teams construct, let only master
+    // AC: skip __kmp_internal_fork at teams construct, let only primary
     // threads execute
     if (ap) {
       __kmp_internal_fork(loc, gtid, team);
       KF_TRACE(10, ("__kmp_internal_fork : after : root=%p, team=%p, "
                     "master_th=%p, gtid=%d\n",
                     root, team, master_th, gtid));
     }
 
     if (call_context == fork_context_gnu) {
       KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid));
       return TRUE;
     }
 
-    /* Invoke microtask for MASTER thread */
+    /* Invoke microtask for PRIMARY thread */
     KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid,
                   team->t.t_id, team->t.t_pkfn));
   } // END of timer KMP_fork_call block
 
 #if KMP_STATS_ENABLED
   // If beginning a teams construct, then change thread state
   stats_state_e previous_state = KMP_GET_THREAD_STATE();
   if (!ap) {
     KMP_SET_THREAD_STATE(stats_state_e::TEAMS_REGION);
   }
 #endif
 
   if (!team->t.t_invoke(gtid)) {
-    KMP_ASSERT2(0, "cannot invoke microtask for MASTER thread");
+    KMP_ASSERT2(0, "cannot invoke microtask for PRIMARY thread");
   }
 
 #if KMP_STATS_ENABLED
   // If was beginning of a teams construct, then reset thread state
   if (!ap) {
     KMP_SET_THREAD_STATE(previous_state);
   }
 #endif

 #endif
     /* Decrement our nested depth level */
     team->t.t_level--;
     team->t.t_active_level--;
     KMP_ATOMIC_DEC(&root->r.r_in_parallel);
 
     // Restore number of threads in the team if needed. This code relies on
     // the proper adjustment of th_teams_size.nth after the fork in
-    // __kmp_teams_master on each teams master in the case that
+    // __kmp_teams_master on each teams primary thread in the case that
     // __kmp_reserve_threads reduced it.
     if (master_th->th.th_team_nproc < master_th->th.th_teams_size.nth) {
       int old_num = master_th->th.th_team_nproc;
       int new_num = master_th->th.th_teams_size.nth;
       kmp_info_t **other_threads = team->t.t_threads;
       team->t.t_nproc = new_num;
       for (int i = 0; i < old_num; ++i) {
         other_threads[i]->th.th_team_nproc = new_num;
__kmp_free_team(root,
                     master_th->th.th_serial_team USE_NESTED_HOT_ARG(NULL));
     master_th->th.th_serial_team = parent_team;
   }
 
   if (__kmp_tasking_mode != tskm_immediate_exec) {
     if (master_th->th.th_task_state_top >
         0) { // Restore task state from memo stack
       KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack);
-      // Remember master's state if we re-use this nested hot team
+      // Remember primary thread's state if we re-use this nested hot team
       master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top] =
           master_th->th.th_task_state;
       --master_th->th.th_task_state_top; // pop
       // Now restore state at this level
       master_th->th.th_task_state =
           master_th->th
               .th_task_state_memo_stack[master_th->th.th_task_state_top];
     }
-    // Copy the task team from the parent team to the master thread
+    // Copy the task team from the parent team to the primary thread
     master_th->th.th_task_team =
         parent_team->t.t_task_team[master_th->th.th_task_state];
     KA_TRACE(20,
-             ("__kmp_join_call: Master T#%d restoring task_team %p / team %p\n",
+             ("__kmp_join_call: Primary T#%d restoring task_team %p, team %p\n",
               __kmp_gtid_from_thread(master_th), master_th->th.th_task_team,
               parent_team));
   }
 
   // TODO: GEH - cannot do this assertion because root thread not set up as
   // executing
   // KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 0 );
   master_th->th.th_current_task->td_flags.executing = 1;
root_team =
       __kmp_allocate_team(root,
                           1, // new_nproc
                           1, // max_nproc
 #if OMPT_SUPPORT
                           ompt_data_none, // root parallel id
 #endif
                           __kmp_nested_proc_bind.bind_types[0], &r_icvs,
                           0 // argc
-                          USE_NESTED_HOT_ARG(NULL) // master thread is unknown
+                          USE_NESTED_HOT_ARG(NULL) // primary thread is unknown
                           );
 #if USE_DEBUGGER
   // Non-NULL value should be assigned to make the debugger display the root
   // team.
   TCW_SYNC_PTR(root_team->t.t_pkfn, (microtask_t)(~0));
 #endif
 
   KF_TRACE(10, ("__kmp_initialize_root: after root_team = %p\n", root_team));
hot_team =
       __kmp_allocate_team(root,
                           1, // new_nproc
                           __kmp_dflt_team_nth_ub * 2, // max_nproc
 #if OMPT_SUPPORT
                           ompt_data_none, // root parallel id
 #endif
                           __kmp_nested_proc_bind.bind_types[0], &r_icvs,
                           0 // argc
-                          USE_NESTED_HOT_ARG(NULL) // master thread is unknown
+                          USE_NESTED_HOT_ARG(NULL) // primary thread is unknown
                           );
   KF_TRACE(10, ("__kmp_initialize_root: after hot_team = %p\n", hot_team));
 
   root->r.r_hot_team = hot_team;
   root_team->t.t_control_stack_top = NULL;
 
   /* first-time initialization */
   hot_team->t.t_parent = root_team;
for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
       kmp_info_t const *thread = __kmp_threads[gtid];
       if (thread != NULL) {
         __kmp_printf("GTID %2d %p:\n", gtid, thread);
         __kmp_printf("    Our Root:        %p\n", thread->th.th_root);
         __kmp_print_structure_team("    Our Team:     ", thread->th.th_team);
         __kmp_print_structure_team("    Serial Team:  ",
                                    thread->th.th_serial_team);
         __kmp_printf("    Threads:      %2d\n", thread->th.th_team_nproc);
-        __kmp_print_structure_thread("    Master:       ",
+        __kmp_print_structure_thread("    Primary:      ",
                                      thread->th.th_team_master);
         __kmp_printf("    Serialized?:  %2d\n", thread->th.th_team_serialized);
         __kmp_printf("    Set NProc:    %2d\n", thread->th.th_set_nproc);
         __kmp_printf("    Set Proc Bind: %2d\n", thread->th.th_set_proc_bind);
         __kmp_print_structure_thread("    Next in pool: ",
                                      thread->th.th_next_pool);
         __kmp_printf("\n");
         __kmp_print_structure_team_accum(list, thread->th.th_team);
void __kmp_print_structure(void) {
 
   __kmp_printf("\n------------------------------\nTeams\n----------------------"
                "--------\n");
   while (list->next != NULL) {
     kmp_team_p const *team = list->entry;
     int i;
     __kmp_printf("Team %2x %p:\n", team->t.t_id, team);
     __kmp_print_structure_team("    Parent Team:      ", team->t.t_parent);
-    __kmp_printf("    Master TID:       %2d\n", team->t.t_master_tid);
+    __kmp_printf("    Primary TID:      %2d\n", team->t.t_master_tid);
     __kmp_printf("    Max threads:      %2d\n", team->t.t_max_nproc);
     __kmp_printf("    Levels of serial: %2d\n", team->t.t_serialized);
     __kmp_printf("    Number threads:   %2d\n", team->t.t_nproc);
     for (i = 0; i < team->t.t_nproc; ++i) {
       __kmp_printf("    Thread %2d:      ", i);
       __kmp_print_structure_thread("", team->t.t_threads[i]);
     }
     __kmp_print_structure_team("    Next in pool:     ", team->t.t_next_pool);
#endif
   // AC: the team created in reserve, not for execution (it is unused for now).
   root_thread->th.th_serial_team->t.t_serialized = 0;
   root->r.r_uber_thread = root_thread;
 
   /* initialize the thread, get it ready to go */
   __kmp_initialize_info(root_thread, root->r.r_root_team, 0, gtid);
   TCW_4(__kmp_init_gtid, TRUE);
 
-  /* prepare the master thread for get_gtid() */
+  /* prepare the primary thread for get_gtid() */
   __kmp_gtid_set_specific(gtid);
 
 #if USE_ITT_BUILD
   __kmp_itt_thread_name(gtid);
 #endif /* USE_ITT_BUILD */
 
 #ifdef KMP_TDATA_GTID
   __kmp_gtid = gtid;
static int __kmp_free_hot_teams(kmp_root_t *root, kmp_info_t *thr, int level,
   int i, n, nth;
   kmp_hot_team_ptr_t *hot_teams = thr->th.th_hot_teams;
   if (!hot_teams || !hot_teams[level].hot_team) {
     return 0;
   }
   KMP_DEBUG_ASSERT(level < max_level);
   kmp_team_t *team = hot_teams[level].hot_team;
   nth = hot_teams[level].hot_team_nth;
-  n = nth - 1; // master is not freed
+  n = nth - 1; // primary thread is not freed
   if (level < max_level - 1) {
     for (i = 0; i < nth; ++i) {
       kmp_info_t *th = team->t.t_threads[i];
       n += __kmp_free_hot_teams(root, th, level + 1, max_level);
       if (i > 0 && th->th.th_hot_teams) {
         __kmp_free(th->th.th_hot_teams);
         th->th.th_hot_teams = NULL;
       }
if (!this_thr->th.th_pri_common) {
     if (__kmp_storage_map) {
       __kmp_print_storage_map_gtid(
           gtid, this_thr->th.th_pri_common, this_thr->th.th_pri_common + 1,
           sizeof(struct common_table), "th_%d.th_pri_common\n", gtid);
     }
     this_thr->th.th_pri_head = NULL;
   }
 
-  if (this_thr != master && // Master's CG root is initialized elsewhere
+  if (this_thr != master && // Primary thread's CG root is initialized elsewhere
       this_thr->th.th_cg_roots != master->th.th_cg_roots) { // CG root not set
-    // Make new thread's CG root same as master's
+    // Make new thread's CG root same as primary thread's
     KMP_DEBUG_ASSERT(master->th.th_cg_roots);
     kmp_cg_root_t *tmp = this_thr->th.th_cg_roots;
     if (tmp) {
       // worker changes CG, need to check if old CG should be freed
       int i = tmp->cg_nthreads--;
       KA_TRACE(100, ("__kmp_initialize_info: Thread %p decrement cg_nthreads"
                      " on node %p of thread %p to %d\n",
                      this_thr, tmp, tmp->cg_root, tmp->cg_nthreads));
if (!TCR_4(__kmp_init_monitor)) {
       TCW_4(__kmp_init_monitor, 1);
       __kmp_create_monitor(&__kmp_monitor);
       KF_TRACE(10, ("after __kmp_create_monitor\n"));
 #if KMP_OS_WINDOWS
       // AC: wait until monitor has started. This is a fix for CQ232808.
       // The reason is that if the library is loaded/unloaded in a loop with
       // small (parallel) work in between, then there is high probability that
       // monitor thread started after the library shutdown. At shutdown it is
-      // too late to cope with the problem, because when the master is in
-      // DllMain (process detach) the monitor has no chances to start (it is
-      // blocked), and master has no means to inform the monitor that the
-      // library has gone, because all the memory which the monitor can access
-      // is going to be released/reset.
+      // too late to cope with the problem, because when the primary thread is
+      // in DllMain (process detach) the monitor has no chances to start (it is
+      // blocked), and primary thread has no means to inform the monitor that
+      // the library has gone, because all the memory which the monitor can
+      // access is going to be released/reset.
       while (TCR_4(__kmp_init_monitor) < 2) {
         KMP_YIELD(TRUE);
       }
       KF_TRACE(10, ("after monitor thread has started\n"));
 #endif
     }
     __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
   }
__itt_suppress_mark_range(__itt_suppress_range,
                             __itt_suppress_threading_errors,
                             CCAST(kmp_uint64 *, &new_thr->th.th_bar[2].bb.b_go),
                             sizeof(new_thr->th.th_bar[2].bb.b_go));
 #endif /* USE_ITT_BUILD && USE_ITT_NOTIFY && KMP_DEBUG */
   if (__kmp_storage_map) {
     __kmp_print_thread_storage_map(new_thr, new_gtid);
   }
 
-  // add the reserve serialized team, initialized from the team's master thread
+  // add the reserve serialized team, initialized from the team's primary thread
   {
     kmp_internal_control_t r_icvs = __kmp_get_x_global_icvs(team);
     KF_TRACE(10, ("__kmp_allocate_thread: before th_serial/serial_team\n"));
     new_thr->th.th_serial_team = serial_team =
         (kmp_team_t *)__kmp_allocate_team(root, 1, 1,
 #if OMPT_SUPPORT
                                           ompt_data_none, // root parallel id
 #endif
static void __kmp_reinitialize_team(kmp_team_t *team,
                                     ident_t *loc) {
   KF_TRACE(10, ("__kmp_reinitialize_team: enter this_thread=%p team=%p\n",
                 team->t.t_threads[0], team));
   KMP_DEBUG_ASSERT(team && new_icvs);
   KMP_DEBUG_ASSERT((!TCR_4(__kmp_init_parallel)) || new_icvs->nproc);
   KMP_CHECK_UPDATE(team->t.t_ident, loc);
 
   KMP_CHECK_UPDATE(team->t.t_id, KMP_GEN_TEAM_ID());
-  // Copy ICVs to the master thread's implicit taskdata
+  // Copy ICVs to the primary thread's implicit taskdata
   __kmp_init_implicit_task(loc, team->t.t_threads[0], team, 0, FALSE);
   copy_icvs(&team->t.t_implicit_task_taskdata[0].td_icvs, new_icvs);
 
   KF_TRACE(10, ("__kmp_reinitialize_team: exit this_thread=%p team=%p\n",
                 team->t.t_threads[0], team));
 }
 
 /* Initialize the team data structure.
if (KMP_AFFINITY_CAPABLE()) {
     __kmp_set_system_affinity(__kmp_affin_fullMask, TRUE);
   }
 }
 #endif
 
 #if KMP_AFFINITY_SUPPORTED
 
 // __kmp_partition_places() is the heart of the OpenMP 4.0 affinity mechanism.
-// It calculates the worker + master thread's partition based upon the parent
+// It calculates the worker + primary thread's partition based upon the parent
 // thread's partition, and binds each worker to a thread in their partition.
-// The master thread's partition should already include its current binding.
+// The primary thread's partition should already include its current binding.
 static void __kmp_partition_places(kmp_team_t *team, int update_master_only) {
-  // Copy the master thread's place partition to the team struct
+  // Copy the primary thread's place partition to the team struct
   kmp_info_t *master_th = team->t.t_threads[0];
   KMP_DEBUG_ASSERT(master_th != NULL);
   kmp_proc_bind_t proc_bind = team->t.t_proc_bind;
   int first_place = master_th->th.th_first_place;
   int last_place = master_th->th.th_last_place;
   int masters_place = master_th->th.th_current_place;
   team->t.t_first_place = first_place;
   team->t.t_last_place = last_place;
 
   KA_TRACE(20, ("__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) "
                 "bound to place %d partition = [%d,%d]\n",
                 proc_bind, __kmp_gtid_from_thread(team->t.t_threads[0]),
                 team->t.t_id, masters_place, first_place, last_place));
 
   switch (proc_bind) {
 
   case proc_bind_default:
-    // serial teams might have the proc_bind policy set to proc_bind_default. It
-    // doesn't matter, as we don't rebind master thread for any proc_bind policy
+    // Serial teams might have the proc_bind policy set to proc_bind_default.
+    // Not an issue -- we don't rebind primary thread for any proc_bind policy.
     KMP_DEBUG_ASSERT(team->t.t_nproc == 1);
     break;
 
-  case proc_bind_master: {
+  case proc_bind_primary: {
     int f;
     int n_th = team->t.t_nproc;
     for (f = 1; f < n_th; f++) {
       kmp_info_t *th = team->t.t_threads[f];
       KMP_DEBUG_ASSERT(th != NULL);
       th->th.th_first_place = first_place;
       th->th.th_last_place = last_place;
       th->th.th_new_place = masters_place;
       if (__kmp_display_affinity && masters_place != th->th.th_current_place &&
           team->t.t_display_affinity != 1) {
         team->t.t_display_affinity = 1;
       }
 
-      KA_TRACE(100, ("__kmp_partition_places: master: T#%d(%d:%d) place %d "
+      KA_TRACE(100, ("__kmp_partition_places: primary: T#%d(%d:%d) place %d "
                      "partition = [%d,%d]\n",
                      __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id,
                      f, masters_place, first_place, last_place));
     }
   } break;
 
   case proc_bind_close: {
     int f;
if (team->t.t_nproc == new_nproc) { // Check changes in number of threads
       if (team->t.t_size_changed == -1) {
         team->t.t_size_changed = 1;
       } else {
         KMP_CHECK_UPDATE(team->t.t_size_changed, 0);
       }
 
       // TODO???: team->t.t_max_active_levels = new_max_active_levels;
       kmp_r_sched_t new_sched = new_icvs->sched;
-      // set master's schedule as new run-time schedule
+      // set primary thread's schedule as new run-time schedule
       KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched);
 
       __kmp_reinitialize_team(team, new_icvs,
                               root->r.r_uber_thread->th.th_ident);
 
       KF_TRACE(10, ("__kmp_allocate_team2: T#%d, this_thread=%p team=%p\n", 0,
                     team->t.t_threads[0], team));
       __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0);
#endif // KMP_NESTED_HOT_TEAMS
       __kmp_reinitialize_team(team, new_icvs,
                               root->r.r_uber_thread->th.th_ident);
 
       // Update remaining threads
       for (f = 0; f < new_nproc; ++f) {
         team->t.t_threads[f]->th.th_team_nproc = new_nproc;
       }
 
-      // restore the current task state of the master thread: should be the
+      // restore the current task state of the primary thread: should be the
       // implicit task
       KF_TRACE(10, ("__kmp_allocate_team: T#%d, this_thread=%p team=%p\n", 0,
                     team->t.t_threads[0], team));
 
       __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0);
 
 #ifdef KMP_DEBUG
       for (f = 0; f < team->t.t_nproc; f++) {

 #endif // KMP_NESTED_HOT_TEAMS
         if (team->t.t_max_nproc < new_nproc) {
           /* reallocate larger arrays */
           __kmp_reallocate_team_arrays(team, new_nproc);
           __kmp_reinitialize_team(team, new_icvs, NULL);
         }
 
 #if (KMP_OS_LINUX || KMP_OS_FREEBSD) && KMP_AFFINITY_SUPPORTED
-        /* Temporarily set full mask for master thread before creation of
-           workers. The reason is that workers inherit the affinity from master,
-           so if a lot of workers are created on the single core quickly, they
-           don't get a chance to set their own affinity for a long time. */
+        /* Temporarily set full mask for primary thread before creation of
+           workers. The reason is that workers inherit the affinity from the
+           primary thread, so if a lot of workers are created on the single
+           core quickly, they don't get a chance to set their own affinity for
+           a long time. */
         __kmp_set_thread_affinity_mask_full_tmp(old_mask);
 #endif
 
         /* allocate new threads for the hot team */
         for (f = team->t.t_nproc; f < new_nproc; f++) {
           kmp_info_t *new_worker = __kmp_allocate_thread(root, team, f);
           KMP_DEBUG_ASSERT(new_worker);
           team->t.t_threads[f] = new_worker;
#if USE_DEBUGGER
               balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
 #endif
             }
           }
         }
 
 #if (KMP_OS_LINUX || KMP_OS_FREEBSD) && KMP_AFFINITY_SUPPORTED
         if (KMP_AFFINITY_CAPABLE()) {
-          /* Restore initial master thread's affinity mask */
+          /* Restore initial primary thread's affinity mask */
           __kmp_set_system_affinity(old_mask, TRUE);
           KMP_CPU_FREE(old_mask);
         }
 #endif
 #if KMP_NESTED_HOT_TEAMS
       } // end of check of t_nproc vs. new_nproc vs. hot_team_nth
 #endif // KMP_NESTED_HOT_TEAMS
       /* make sure everyone is syncronized */
       int old_nproc = team->t.t_nproc; // save old value and use to update only
       // new threads below
       __kmp_initialize_team(team, new_nproc, new_icvs,
                             root->r.r_uber_thread->th.th_ident);
 
       /* reinitialize the threads */
       KMP_DEBUG_ASSERT(team->t.t_nproc == new_nproc);
       for (f = 0; f < team->t.t_nproc; ++f)
         __kmp_initialize_info(team->t.t_threads[f], team, f,
                               __kmp_gtid_from_tid(f, team));
 
       if (level) { // set th_task_state for new threads in nested hot team
         // __kmp_initialize_info() no longer zeroes th_task_state, so we should
         // only need to set the th_task_state for the new threads. th_task_state
-        // for master thread will not be accurate until after this in
-        // __kmp_fork_call(), so we look to the master's memo_stack to get the
-        // correct value.
+        // for primary thread will not be accurate until after this in
+        // __kmp_fork_call(), so we look to the primary thread's memo_stack to
+        // get the correct value.
         for (f = old_nproc; f < team->t.t_nproc; ++f)
           team->t.t_threads[f]->th.th_task_state =
               team->t.t_threads[0]->th.th_task_state_memo_stack[level];
       } else { // set th_task_state for new threads in non-nested hot team
-        kmp_uint8 old_state =
-            team->t.t_threads[0]->th.th_task_state; // copy master's state
+        // copy primary thread's state
+        kmp_uint8 old_state = team->t.t_threads[0]->th.th_task_state;
         for (f = old_nproc; f < team->t.t_nproc; ++f)
           team->t.t_threads[f]->th.th_task_state = old_state;
       }
 
 #ifdef KMP_DEBUG
       for (f = 0; f < team->t.t_nproc; ++f) {
         KMP_DEBUG_ASSERT(team->t.t_threads[f] &&
                          team->t.t_threads[f]->th.th_team_nproc ==
for (f = 1; f < team->t.t_nproc; ++f) {
       __kmp_free_thread(team->t.t_threads[f]);
       team->t.t_threads[f] = NULL;
     }
 
     /* put the team back in the team pool */
     /* TODO limit size of team pool, call reap_team if pool too large */
     team->t.t_next_pool = CCAST(kmp_team_t *, __kmp_team_pool);
     __kmp_team_pool = (volatile kmp_team_t *)team;
-  } else { // Check if team was created for the masters in a teams construct
+  } else { // Check if team was created for primary threads in teams construct
     // See if first worker is a CG root
     KMP_DEBUG_ASSERT(team->t.t_threads[1] &&
                      team->t.t_threads[1]->th.th_cg_roots);
     if (team->t.t_threads[1]->th.th_cg_roots->cg_root == team->t.t_threads[1]) {
       // Clean up the CG root nodes on workers so that this team can be re-used
       for (f = 1; f < team->t.t_nproc; ++f) {
         kmp_info_t *thr = team->t.t_threads[f];
         KMP_DEBUG_ASSERT(thr && thr->th.th_cg_roots &&
#if USE_ITT_BUILD
   }
 #endif /* USE_ITT_BUILD */
   __kmp_run_after_invoked_task(gtid, tid, this_thr, team);
 
   return rc;
 }
 
 void __kmp_teams_master(int gtid) {
-  // This routine is called by all master threads in teams construct
+  // This routine is called by all primary threads in teams construct
   kmp_info_t *thr = __kmp_threads[gtid];
   kmp_team_t *team = thr->th.th_team;
   ident_t *loc = team->t.t_ident;
   thr->th.th_set_nproc = thr->th.th_teams_size.nth;
   KMP_DEBUG_ASSERT(thr->th.th_teams_microtask);
   KMP_DEBUG_ASSERT(thr->th.th_set_nproc);
   KA_TRACE(20, ("__kmp_teams_master: T#%d, Tid %d, microtask %p\n", gtid,
                 __kmp_tid_from_gtid(gtid), thr->th.th_teams_microtask));
 
   // This thread is a new CG root.  Set up the proper variables.
   kmp_cg_root_t *tmp = (kmp_cg_root_t *)__kmp_allocate(sizeof(kmp_cg_root_t));
   tmp->cg_root = thr; // Make thr the CG root
-  // Init to thread limit that was stored when league masters were forked
+  // Init to thread limit stored when league primary threads were forked
   tmp->cg_thread_limit = thr->th.th_current_task->td_icvs.thread_limit;
   tmp->cg_nthreads = 1; // Init counter to one active thread, this one
   KA_TRACE(100, ("__kmp_teams_master: Thread %p created node %p and init"
                  " cg_nthreads to 1\n",
                  thr, tmp));
   tmp->up = thr->th.th_cg_roots;
   thr->th.th_cg_roots = tmp;
 
if (num_threads == 0) {
     } // prevent team size to exceed thread-limit-var
     if (num_teams * num_threads > __kmp_teams_max_nth) {
       num_threads = __kmp_teams_max_nth / num_teams;
     }
     if (num_threads == 0) {
       num_threads = 1;
     }
   } else {
-    // This thread will be the master of the league masters
+    // This thread will be the primary thread of the league primary threads
     // Store new thread limit; old limit is saved in th_cg_roots list
     thr->th.th_current_task->td_icvs.thread_limit = num_threads;
     // num_threads = min(num_threads, nthreads-var)
     if (num_threads > __kmp_dflt_team_nth) {
       num_threads = __kmp_dflt_team_nth; // honor nthreads-var ICV
     }
     if (num_teams * num_threads > __kmp_teams_max_nth) {
       int new_threads = __kmp_teams_max_nth / num_teams;
static int __kmp_active_hot_team_nproc(kmp_root_t *root) {
   int retval;
   kmp_team_t *hot_team;
 
   if (root->r.r_active) {
     return 0;
   }
   hot_team = root->r.r_hot_team;
   if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
-    return hot_team->t.t_nproc - 1; // Don't count master thread
+    return hot_team->t.t_nproc - 1; // Don't count primary thread
   }
 
-  // Skip the master thread - it is accounted for elsewhere.
+  // Skip the primary thread - it is accounted for elsewhere.
   retval = 0;
   for (i = 1; i < hot_team->t.t_nproc; i++) {
     if (hot_team->t.t_threads[i]->th.th_active) {
       retval++;
     }
   }
   return retval;
 }
static int __kmp_load_balance_nproc(kmp_root_t *root, int set_nproc) {
 
   if (set_nproc == 1) {
     KB_TRACE(20, ("__kmp_load_balance_nproc: serial execution.\n"));
     return 1;
   }
 
   // Threads that are active in the thread pool, active in the hot team for this
   // particular root (if we are at the outer par level), and the currently
-  // executing thread (to become the master) are available to add to the new
-  // team, but are currently contributing to the system load, and must be
+  // executing thread (to become the primary thread) are available to add to the
+  // new team, but are currently contributing to the system load, and must be
   // accounted for.
   pool_active = __kmp_thread_pool_active_nth;
   hot_team_active = __kmp_active_hot_team_nproc(root);
   team_curr_active = pool_active + hot_team_active + 1;
 
   // Check the system load.
   system_active = __kmp_get_load_balance(__kmp_avail_proc + team_curr_active);
   KB_TRACE(30, ("__kmp_load_balance_nproc: system active = %d pool active = %d "

openmp/runtime/src/kmp_sched.cpp

#endif
   }
 
   *pstride = *pupper - *plower; // just in case (can be unused)
   if (trip_count <= nteams) {
     KMP_DEBUG_ASSERT(
         __kmp_static == kmp_sch_static_greedy ||
         __kmp_static ==
             kmp_sch_static_balanced); // Unknown static scheduling type.
-    // only masters of some teams get single iteration, other threads get
-    // nothing
+    // only primary threads of some teams get single iteration, other threads
+    // get nothing
     if (team_id < trip_count && tid == 0) {
       *pupper = *pupperDist = *plower = *plower + team_id * incr;
     } else {
       *pupperDist = *pupper;
       *plower = *pupper + incr; // compiler should skip loop body
     }
     if (plastiter != NULL)
       *plastiter = (tid == 0 && team_id == trip_count - 1);

openmp/runtime/src/kmp_settings.cpp

#endif /* KMP_AFFINITY_SUPPORTED */
     if (nelem > 1 && !__kmp_dflt_max_active_levels_set)
       __kmp_dflt_max_active_levels = KMP_MAX_ACTIVE_LEVELS_LIMIT;
 
     // Save values in the nested proc_bind array
     int i = 0;
     for (;;) {
       enum kmp_proc_bind_t bind;
 
-      if ((num == (int)proc_bind_master) ||
-          __kmp_match_str("master", buf, &next)) {
+      if ((num == (int)proc_bind_primary) ||
+          __kmp_match_str("master", buf, &next) ||
+          __kmp_match_str("primary", buf, &next)) {
         buf = next;
         SKIP_WS(buf);
-        bind = proc_bind_master;
+        bind = proc_bind_primary;
       } else if ((num == (int)proc_bind_close) ||
                  __kmp_match_str("close", buf, &next)) {
         buf = next;
         SKIP_WS(buf);
         bind = proc_bind_close;
       } else if ((num == (int)proc_bind_spread) ||
                  __kmp_match_str("spread", buf, &next)) {
         buf = next;
for (i = 0; i < nelem; i++) {
       case proc_bind_false:
         __kmp_str_buf_print(buffer, "false");
         break;
 
       case proc_bind_true:
         __kmp_str_buf_print(buffer, "true");
         break;
 
-      case proc_bind_master:
-        __kmp_str_buf_print(buffer, "master");
+      case proc_bind_primary:
+        __kmp_str_buf_print(buffer, "primary");
         break;
 
       case proc_bind_close:
         __kmp_str_buf_print(buffer, "close");
         break;
 
       case proc_bind_spread:
         __kmp_str_buf_print(buffer, "spread");

openmp/runtime/src/kmp_stats.h

 
 /*!
  * @ingroup STATS_GATHERING
  * \brief flags to describe the statistic (timer or counter)
  *
  */
 enum stats_flags_e {
   noTotal = 1 << 0, //!< do not show a TOTAL_aggregation for this statistic
-  onlyInMaster = 1 << 1, //!< statistic is valid only for master
+  onlyInMaster = 1 << 1, //!< statistic is valid only for primary thread
   noUnits = 1 << 2, //!< statistic doesn't need units printed next to it
-  notInMaster = 1 << 3, //!< statistic is valid only for non-master threads
+  notInMaster = 1 << 3, //!< statistic is valid only for non-primary threads
   logEvent = 1 << 4 //!< statistic can be logged on the event timeline when
   //! KMP_STATS_EVENTS is on (valid only for timers)
 };
 
 /*!
  * @ingroup STATS_GATHERING
  * \brief the states which a thread can be in
  *
#define KMP_FOREACH_COUNTER(macro, arg)                                        \
   macro(OMP_LOOP_STATIC, 0, arg)                                               \
   macro(OMP_LOOP_STATIC_STEAL, 0, arg)                                         \
   macro(OMP_LOOP_DYNAMIC, 0, arg)                                              \
   macro(OMP_DISTRIBUTE, 0, arg)                                                \
   macro(OMP_BARRIER, 0, arg)                                                   \
   macro(OMP_CRITICAL, 0, arg)                                                  \
   macro(OMP_SINGLE, 0, arg)                                                    \
   macro(OMP_MASTER, 0, arg)                                                    \
+  macro(OMP_MASKED, 0, arg)                                                    \
   macro(OMP_TEAMS, 0, arg)                                                     \
   macro(OMP_set_lock, 0, arg)                                                  \
   macro(OMP_test_lock, 0, arg)                                                 \
   macro(REDUCE_wait, 0, arg)                                                   \
   macro(REDUCE_nowait, 0, arg)                                                 \
   macro(OMP_TASKYIELD, 0, arg)                                                 \
   macro(OMP_TASKLOOP, 0, arg)                                                  \
   macro(TASK_executed, 0, arg)                                                 \
#define KMP_FOREACH_TIMER(macro, arg)                                          \
   macro (OMP_loop_dynamic, 0, arg)                                             \
   macro (OMP_loop_dynamic_scheduling, 0, arg)                                  \
   macro (OMP_distribute, 0, arg)                                               \
   macro (OMP_distribute_scheduling, 0, arg)                                    \
   macro (OMP_critical, 0, arg)                                                 \
   macro (OMP_critical_wait, 0, arg)                                            \
   macro (OMP_single, 0, arg)                                                   \
   macro (OMP_master, 0, arg)                                                   \
+  macro (OMP_masked, 0, arg)                                                   \
   macro (OMP_task_immediate, 0, arg)                                           \
   macro (OMP_task_taskwait, 0, arg)                                            \
   macro (OMP_task_taskyield, 0, arg)                                           \
   macro (OMP_task_taskgroup, 0, arg)                                           \
   macro (OMP_task_join_bar, 0, arg)                                            \
   macro (OMP_task_plain_bar, 0, arg)                                           \
   macro (OMP_taskloop_scheduling, 0, arg)                                      \
   macro (OMP_plain_barrier, stats_flags_e::logEvent, arg)                      \
#define KMP_FOREACH_TIMER(macro, arg)                                          \
   macro (OMP_loop_dynamic_total_iterations,                                    \
          stats_flags_e::noUnits | stats_flags_e::noTotal, arg)                 \
   macro (OMP_distribute_iterations,                                            \
          stats_flags_e::noUnits | stats_flags_e::noTotal, arg)                 \
   KMP_FOREACH_DEVELOPER_TIMER(macro, arg)
 // clang-format on
 
 // OMP_worker_thread_life -- Time from thread becoming an OpenMP thread (either
-//                           initializing OpenMP or being created by a master)
-//                           until the thread is destroyed
+//                           initializing OpenMP or being created by a primary
+//                           thread) until the thread is destroyed
 // OMP_parallel           -- Time thread spends executing work directly
 //                           within a #pragma omp parallel
 // OMP_parallel_overhead  -- Time thread spends setting up a parallel region
 // OMP_loop_static        -- Time thread spends executing loop iterations from
 //                           a statically scheduled loop
 // OMP_loop_static_scheduling -- Time thread spends scheduling loop iterations
 //                               from a statically scheduled loop
 // OMP_loop_dynamic       -- Time thread spends executing loop iterations from
 //                           a dynamically scheduled loop
 // OMP_loop_dynamic_scheduling -- Time thread spends scheduling loop iterations
 //                                from a dynamically scheduled loop
 // OMP_critical           -- Time thread spends executing critical section
 // OMP_critical_wait      -- Time thread spends waiting to enter
 //                           a critical section
 // OMP_single             -- Time spent executing a "single" region
 // OMP_master             -- Time spent executing a "master" region
+// OMP_masked             -- Time spent executing a "masked" region
 // OMP_task_immediate     -- Time spent executing non-deferred tasks
 // OMP_task_taskwait      -- Time spent executing tasks inside a taskwait
 //                           construct
 // OMP_task_taskyield     -- Time spent executing tasks inside a taskyield
 //                           construct
 // OMP_task_taskgroup     -- Time spent executing tasks inside a taskygroup
 //                           construct
 // OMP_task_join_bar      -- Time spent executing tasks inside a join barrier
/* ****************************************************************
     set up in the constructor. Each node also has an event vector to hold that
     thread's timing events. The event vector expands as necessary and records
     the start-stop times for each timer.
 
     The nestLevel variable is for plotting events and is related
     to the bar width in the timeline graph.
 
     Every thread will have a thread local pointer to its node in
-    the list.  The sentinel node is used by the master thread to
+    the list.  The sentinel node is used by the primary thread to
     store "dummy" statistics before __kmp_create_worker() is called.
 **************************************************************** */
 class kmp_stats_list {
   int gtid;
   timeStat _timers[TIMER_LAST + 1];
   counter _counters[COUNTER_LAST + 1];
   explicitTimer thread_life_timer;
   partitionedTimers _partitionedTimers;

openmp/runtime/src/kmp_stats.cpp

for (it = __kmp_stats_list->begin(); it != __kmp_stats_list->end(); it++) {
     if (eventPrintingEnabled()) {
       kmp_stats_event_vector events = (*it)->getEventVector();
       printEvents(eventsOut, &events, t);
     }
 
     // Accumulate timers.
     for (timer_e s = timer_e(0); s < TIMER_LAST; s = timer_e(s + 1)) {
       // See if we should ignore this timer when aggregating
-      if ((timeStat::masterOnly(s) && (t != 0)) || // Timer only valid on master
-          // and this thread is worker
+      if ((timeStat::masterOnly(s) && (t != 0)) || // Timer only valid on
+          // primary thread and this thread is worker
           (timeStat::workerOnly(s) && (t == 0)) // Timer only valid on worker
-          // and this thread is the master
+          // and this thread is the primary thread
           ) {
         continue;
       }
 
       statistic *threadStat = (*it)->getTimer(s);
       allStats[s] += *threadStat;
 
       // Add Total stats for timers that are valid in more than one thread

openmp/runtime/src/kmp_tasking.cpp

if (__kmp_task_is_allowed(gtid, is_constrained, taskdata, current)) {
     KMP_DEBUG_ASSERT(
         victim_td->td.td_deque_tail ==
         (kmp_uint32)((target + 1) & TASK_DEQUE_MASK(victim_td->td)));
     victim_td->td.td_deque_tail = target; // tail -= 1 (wrapped))
   }
   if (*thread_finished) {
     // We need to un-mark this victim as a finished victim.  This must be done
     // before releasing the lock, or else other threads (starting with the
-    // master victim) might be prematurely released from the barrier!!!
+    // primary thread victim) might be prematurely released from the barrier!!!
     kmp_int32 count;
 
     count = KMP_ATOMIC_INC(unfinished_threads);
 
     KA_TRACE(
         20,
         ("__kmp_steal_task: T#%d inc unfinished_threads to %d: task_team=%p\n",
          gtid, count + 1, task_team));
if (final_spin &&
         count = KMP_ATOMIC_DEC(unfinished_threads) - 1;
         KA_TRACE(20, ("__kmp_execute_tasks_template: T#%d dec "
                       "unfinished_threads to %d task_team=%p\n",
                       gtid, count, task_team));
         *thread_finished = TRUE;
       }
 
       // It is now unsafe to reference thread->th.th_team !!!
-      // Decrementing task_team->tt.tt_unfinished_threads can allow the master
+      // Decrementing task_team->tt.tt_unfinished_threads can allow the primary
       // thread to pass through the barrier, where it might reset each thread's
       // th.th_team field for the next parallel region. If we can steal more
       // work, we know that this has not happened yet.
       if (flag != NULL && flag->done_check()) {
         KA_TRACE(
             15,
             ("__kmp_execute_tasks_template: T#%d spin condition satisfied\n",
              gtid));
         return TRUE;
       }
     }
 
-    // If this thread's task team is NULL, master has recognized that there are
-    // no more tasks; bail out
+    // If this thread's task team is NULL, primary thread has recognized that
+    // there are no more tasks; bail out
     if (thread->th.th_task_team == NULL) {
       KA_TRACE(15,
                ("__kmp_execute_tasks_template: T#%d no more tasks\n", gtid));
       return FALSE;
     }
 
     // We could be getting tasks from target constructs; if this is the only
     // thread, keep trying to execute tasks from own queue

 }
 
 /* // TODO: Check the comment consistency
  * Utility routines for "task teams".  A task team (kmp_task_t) is kind of
  * like a shadow of the kmp_team_t data struct, with a different lifetime.
  * After a child * thread checks into a barrier and calls __kmp_release() from
  * the particular variant of __kmp_<barrier_kind>_barrier_gather(), it can no
  * longer assume that the kmp_team_t structure is intact (at any moment, the
- * master thread may exit the barrier code and free the team data structure,
+ * primary thread may exit the barrier code and free the team data structure,
  * and return the threads to the thread pool).
  *
  * This does not work with the tasking code, as the thread is still
  * expected to participate in the execution of any tasks that may have been
  * spawned my a member of the team, and the thread still needs access to all
  * to each thread in the team, so that it can steal work from it.
  *
  * Enter the existence of the kmp_task_team_t struct.  It employs a reference
- * counting mechanism, and is allocated by the master thread before calling
+ * counting mechanism, and is allocated by the primary thread before calling
  * __kmp_<barrier_kind>_release, and then is release by the last thread to
  * exit __kmp_<barrier_kind>_release at the next barrier.  I.e. the lifetimes
  * of the kmp_task_team_t structs for consecutive barriers can overlap
- * (and will, unless the master thread is the last thread to exit the barrier
+ * (and will, unless the primary thread is the last thread to exit the barrier
  * release phase, which is not typical). The existence of such a struct is
  * useful outside the context of tasking.
  *
  * We currently use the existence of the threads array as an indicator that
  * tasks were spawned since the last barrier.  If the structure is to be
  * useful outside the context of tasking, then this will have to change, but
  * not setting the field minimizes the performance impact of tasking on
  * barriers, when no explicit tasks were spawned (pushed, actually).
void __kmp_task_team_setup(kmp_info_t *this_thr, kmp_team_t *team, int always) {
   // If this task_team hasn't been created yet, allocate it. It will be used in
   // the region after the next.
   // If it exists, it is the current task team and shouldn't be touched yet as
   // it may still be in use.
   if (team->t.t_task_team[this_thr->th.th_task_state] == NULL &&
       (always || team->t.t_nproc > 1)) {
     team->t.t_task_team[this_thr->th.th_task_state] =
         __kmp_allocate_task_team(this_thr, team);
-    KA_TRACE(20, ("__kmp_task_team_setup: Master T#%d created new task_team %p "
-                  "for team %d at parity=%d\n",
+    KA_TRACE(20, ("__kmp_task_team_setup: Primary T#%d created new task_team %p"
+                  " for team %d at parity=%d\n",
                   __kmp_gtid_from_thread(this_thr),
                   team->t.t_task_team[this_thr->th.th_task_state],
                   ((team != NULL) ? team->t.t_id : -1),
                   this_thr->th.th_task_state));
   }
 
   // After threads exit the release, they will call sync, and then point to this
   // other task_team; make sure it is allocated and properly initialized. As
   // threads spin in the barrier release phase, they will continue to use the
   // previous task_team struct(above), until they receive the signal to stop
   // checking for tasks (they can't safely reference the kmp_team_t struct,
-  // which could be reallocated by the master thread). No task teams are formed
+  // which could be reallocated by the primary thread). No task teams are formed
   // for serialized teams.
   if (team->t.t_nproc > 1) {
     int other_team = 1 - this_thr->th.th_task_state;
     if (team->t.t_task_team[other_team] == NULL) { // setup other team as well
       team->t.t_task_team[other_team] =
           __kmp_allocate_task_team(this_thr, team);
-      KA_TRACE(20, ("__kmp_task_team_setup: Master T#%d created second new "
+      KA_TRACE(20, ("__kmp_task_team_setup: Primary T#%d created second new "
                     "task_team %p for team %d at parity=%d\n",
                     __kmp_gtid_from_thread(this_thr),
                     team->t.t_task_team[other_team],
                     ((team != NULL) ? team->t.t_id : -1), other_team));
     } else { // Leave the old task team struct in place for the upcoming region;
       // adjust as needed
       kmp_task_team_t *task_team = team->t.t_task_team[other_team];
       if (!task_team->tt.tt_active ||
           team->t.t_nproc != task_team->tt.tt_nproc) {
         TCW_4(task_team->tt.tt_nproc, team->t.t_nproc);
         TCW_4(task_team->tt.tt_found_tasks, FALSE);
         TCW_4(task_team->tt.tt_found_proxy_tasks, FALSE);
         KMP_ATOMIC_ST_REL(&task_team->tt.tt_unfinished_threads,
                           team->t.t_nproc);
         TCW_4(task_team->tt.tt_active, TRUE);
       }
       // if team size has changed, the first thread to enable tasking will
       // realloc threads_data if necessary
-      KA_TRACE(20, ("__kmp_task_team_setup: Master T#%d reset next task_team "
+      KA_TRACE(20, ("__kmp_task_team_setup: Primary T#%d reset next task_team "
                     "%p for team %d at parity=%d\n",
                     __kmp_gtid_from_thread(this_thr),
                     team->t.t_task_team[other_team],
                     ((team != NULL) ? team->t.t_id : -1), other_team));
     }
   }
 
   // For regular thread, task enabling should be called when the task is going
TCW_PTR(this_thr->th.th_task_team,
           team->t.t_task_team[this_thr->th.th_task_state]);
   KA_TRACE(20,
            ("__kmp_task_team_sync: Thread T#%d task team switched to task_team "
             "%p from Team #%d (parity=%d)\n",
             __kmp_gtid_from_thread(this_thr), this_thr->th.th_task_team,
             ((team != NULL) ? team->t.t_id : -1), this_thr->th.th_task_state));
 }
 
-// __kmp_task_team_wait: Master thread waits for outstanding tasks after the
-// barrier gather phase. Only called by master thread if #threads in team > 1 or
-// if proxy tasks were created.
+// __kmp_task_team_wait: Primary thread waits for outstanding tasks after the
+// barrier gather phase. Only called by primary thread if #threads in team > 1
+// or if proxy tasks were created.
 //
 // wait is a flag that defaults to 1 (see kmp.h), but waiting can be turned off
-// by passing in 0 optionally as the last argument. When wait is zero, master
+// by passing in 0 optionally as the last argument. When wait is zero, primary
 // thread does not wait for unfinished_threads to reach 0.
 void __kmp_task_team_wait(
     kmp_info_t *this_thr,
     kmp_team_t *team USE_ITT_BUILD_ARG(void *itt_sync_obj), int wait) {
   kmp_task_team_t *task_team = team->t.t_task_team[this_thr->th.th_task_state];
 
   KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
   KMP_DEBUG_ASSERT(task_team == this_thr->th.th_task_team);
 
   if ((task_team != NULL) && KMP_TASKING_ENABLED(task_team)) {
     if (wait) {
-      KA_TRACE(20, ("__kmp_task_team_wait: Master T#%d waiting for all tasks "
+      KA_TRACE(20, ("__kmp_task_team_wait: Primary T#%d waiting for all tasks "
                     "(for unfinished_threads to reach 0) on task_team = %p\n",
                     __kmp_gtid_from_thread(this_thr), task_team));
       // Worker threads may have dropped through to release phase, but could
       // still be executing tasks. Wait here for tasks to complete. To avoid
-      // memory contention, only master thread checks termination condition.
+      // memory contention, only primary thread checks termination condition.
       kmp_flag_32<false, false> flag(
           RCAST(std::atomic<kmp_uint32> *,
                 &task_team->tt.tt_unfinished_threads),
           0U);
       flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj));
     }
     // Deactivate the old task team, so that the worker threads will stop
     // referencing it while spinning.
     KA_TRACE(
         20,
-        ("__kmp_task_team_wait: Master T#%d deactivating task_team %p: "
+        ("__kmp_task_team_wait: Primary T#%d deactivating task_team %p: "
          "setting active to false, setting local and team's pointer to NULL\n",
          __kmp_gtid_from_thread(this_thr), task_team));
     KMP_DEBUG_ASSERT(task_team->tt.tt_nproc > 1 ||
                      task_team->tt.tt_found_proxy_tasks == TRUE);
     TCW_SYNC_4(task_team->tt.tt_found_proxy_tasks, FALSE);
     KMP_CHECK_UPDATE(task_team->tt.tt_untied_task_encountered, 0);
     TCW_SYNC_4(task_team->tt.tt_active, FALSE);
     KMP_MB();

openmp/runtime/src/kmp_threadprivate.cpp

if (TCR_4(__kmp_init_common)) {
     TCW_4(__kmp_init_common, FALSE);
 
     for (q = 0; q < KMP_HASH_TABLE_SIZE; ++q) {
       int gtid;
       struct private_common *tn;
       struct shared_common *d_tn;
 
       /* C++ destructors need to be called once per thread before exiting.
-         Don't call destructors for master thread though unless we used copy
+         Don't call destructors for primary thread though unless we used copy
          constructor */
 
       for (d_tn = __kmp_threadprivate_d_table.data[q]; d_tn;
            d_tn = d_tn->next) {
         if (d_tn->is_vec) {
           if (d_tn->dt.dtorv != 0) {
             for (gtid = 0; gtid < __kmp_all_nth; ++gtid) {
               if (__kmp_threads[gtid]) {
#endif
 
   tn->link = __kmp_threads[gtid]->th.th_pri_head;
   __kmp_threads[gtid]->th.th_pri_head = tn;
 
   if ((__kmp_foreign_tp) ? (KMP_INITIAL_GTID(gtid)) : (KMP_UBER_GTID(gtid)))
     return tn;
 
   /* if C++ object with copy constructor, use it;
-   * else if C++ object with constructor, use it for the non-master copies only;
+   * else if C++ object with constructor, use it for the non-primary thread
+     copies only;
    * else use pod_init and memcpy
    *
-   * C++ constructors need to be called once for each non-master thread on
+   * C++ constructors need to be called once for each non-primary thread on
    * allocate
    * C++ copy constructors need to be called once for each thread on allocate */
 
   /* C++ object with constructors/destructors; don't call constructors for
-     master thread though */
+     primary thread though */
   if (d_tn->is_vec) {
     if (d_tn->ct.ctorv != 0) {
       (void)(*d_tn->ct.ctorv)(tn->par_addr, d_tn->vec_len);
     } else if (d_tn->cct.cctorv != 0) {
       (void)(*d_tn->cct.cctorv)(tn->par_addr, d_tn->obj_init, d_tn->vec_len);
     } else if (tn->par_addr != tn->gbl_addr) {
       __kmp_copy_common_data(tn->par_addr, d_tn->pod_init);
     }