Fork performance improvements: submission from Diego C.

[tlwilmar]

Summary

Most of this is modifications to check for differences before updating data fields in team struct. There is also some rearrangement of team struct.

[AndreyChurbanov]

LGTM

[jlpeyton]

LGTM

openmp/trunk/CREDITS.txt

 D: Patches for the ARM architecture and removal of several inconsistencies.
 
 N: Alp Toker
 E: [e-mail]
 D: Making build work for FreeBSD.
 
 N: Cheng Wang
 D: Contributor to testsuite from OpenUH
+
+N: Diego Caballero
+E: [e-mail]
+D: Fork performance improvements

openmp/trunk/runtime/src/kmp.h

 #ifdef BUILD_TV
     struct tv_data        *tv_data;
 #endif
 
     PACKED_REDUCTION_METHOD_T packed_reduction_method; /* stored by __kmpc_reduce*(), used by __kmpc_end_reduce*() */
 
 } kmp_local_t;
 
+#define KMP_CHECK_UPDATE(a, b) if ((a) != (b)) (a) = (b)
+#define KMP_CHECK_UPDATE_SYNC(a, b) if ((a) != (b)) TCW_SYNC_PTR((a), (b))
+
 #define get__blocktime( xteam, xtid )     ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.blocktime)
 #define get__bt_set( xteam, xtid )        ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.bt_set)
 #define get__bt_intervals( xteam, xtid )  ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.bt_intervals)
 
 #define get__nested_2(xteam,xtid)         ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.nested)
 #define get__dynamic_2(xteam,xtid)        ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.dynamic)
 #define get__nproc_2(xteam,xtid)          ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.nproc)
 #define get__sched_2(xteam,xtid)          ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.sched)
struct kmp_taskdata {                                 /* aligned during dynamic allocation       */
     kmp_int32               td_level;                 /* task nesting level                      */
     kmp_int32               td_untied_count;          /* untied task active parts counter        */
     ident_t *               td_ident;                 /* task identifier                         */
                             // Taskwait data.
     ident_t *               td_taskwait_ident;
     kmp_uint32              td_taskwait_counter;
     kmp_int32               td_taskwait_thread;       /* gtid + 1 of thread encountered taskwait */
     KMP_ALIGN_CACHE kmp_internal_control_t  td_icvs;  /* Internal control variables for the task */
-    volatile kmp_uint32     td_allocated_child_tasks;  /* Child tasks (+ current task) not yet deallocated */
+    KMP_ALIGN_CACHE volatile kmp_uint32 td_allocated_child_tasks;  /* Child tasks (+ current task) not yet deallocated */
     volatile kmp_uint32     td_incomplete_child_tasks; /* Child tasks not yet complete */
 #if OMP_40_ENABLED
     kmp_taskgroup_t *       td_taskgroup;         // Each task keeps pointer to its current taskgroup
     kmp_dephash_t *         td_dephash;           // Dependencies for children tasks are tracked from here
     kmp_depnode_t *         td_depnode;           // Pointer to graph node if this task has dependencies
 #endif
 #if OMPT_SUPPORT
     ompt_task_info_t        ompt_task_info;
#if KMP_ARCH_X86 || KMP_ARCH_X86_64
     kmp_int8                 t_pad2b;
     kmp_int16                t_x87_fpu_control_word; // FP control regs
     kmp_uint32               t_mxcsr;
 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
 
     void                    *t_inline_argv[ KMP_INLINE_ARGV_ENTRIES ];
 
     KMP_ALIGN_CACHE kmp_info_t **t_threads;
-    int                      t_max_argc;
+    kmp_taskdata_t *t_implicit_task_taskdata;  // Taskdata for the thread's implicit task
+    int                      t_level;          // nested parallel level
+
+    KMP_ALIGN_CACHE int      t_max_argc;
     int                      t_max_nproc;    // maximum threads this team can handle (dynamicly expandable)
     int                      t_serialized;   // levels deep of serialized teams
     dispatch_shared_info_t  *t_disp_buffer;  // buffers for dispatch system
     int                      t_id;           // team's id, assigned by debugger.
-    int                      t_level;        // nested parallel level
     int                      t_active_level; // nested active parallel level
     kmp_r_sched_t            t_sched;        // run-time schedule for the team
 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED
     int                      t_first_place;  // first & last place in parent thread's partition.
     int                      t_last_place;   // Restore these values to master after par region.
 #endif // OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED
     int t_size_changed; // team size was changed?: 0: no, 1: yes, -1: changed via omp_set_num_threads() call
 
     // Read/write by workers as well -----------------------------------------------------------------------
 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
     // Using CACHE_LINE=64 reduces memory footprint, but causes a big perf regression of epcc 'parallel'
     // and 'barrier' on fxe256lin01. This extra padding serves to fix the performance of epcc 'parallel'
     // and 'barrier' when CACHE_LINE=64. TODO: investigate more and get rid if this padding.
     char dummy_padding[1024];
 #endif
-    KMP_ALIGN_CACHE kmp_taskdata_t *t_implicit_task_taskdata;  // Taskdata for the thread's implicit task
-    kmp_internal_control_t  *t_control_stack_top;  // internal control stack for additional nested teams.
+    KMP_ALIGN_CACHE kmp_internal_control_t *t_control_stack_top;  // internal control stack for additional nested teams.
                                                    // for SERIALIZED teams nested 2 or more levels deep
 #if OMP_40_ENABLED
     kmp_int32                t_cancel_request; // typed flag to store request state of cancellation
 #endif
     int                      t_master_active;  // save on fork, restore on join
     kmp_taskq_t              t_taskq;          // this team's task queue
     void                    *t_copypriv_data;  // team specific pointer to copyprivate data array
     kmp_uint32               t_copyin_counter;

openmp/trunk/runtime/src/kmp_runtime.c

#if OMP_40_ENABLED
                                    proc_bind,
 #endif
                                    &master_th->th.th_current_task->td_icvs, argc
                                    USE_NESTED_HOT_ARG(master_th) );
     }
     KF_TRACE( 10, ( "__kmp_fork_call: after __kmp_allocate_team - team = %p\n", team ) );
 
     /* setup the new team */
-    team->t.t_master_tid = master_tid;
-    team->t.t_master_this_cons = master_this_cons;
-    team->t.t_ident      = loc;
-    team->t.t_parent     = parent_team;
-    TCW_SYNC_PTR(team->t.t_pkfn, microtask);
+    KMP_CHECK_UPDATE(team->t.t_master_tid, master_tid);
+    KMP_CHECK_UPDATE(team->t.t_master_this_cons, master_this_cons);
+    KMP_CHECK_UPDATE(team->t.t_ident, loc);
+    KMP_CHECK_UPDATE(team->t.t_parent, parent_team);
+    KMP_CHECK_UPDATE_SYNC(team->t.t_pkfn, microtask);
 #if OMPT_SUPPORT
-    TCW_SYNC_PTR(team->t.ompt_team_info.microtask, unwrapped_task);
+    KMP_CHECK_UPDATE_SYNC(team->t.ompt_team_info.microtask, unwrapped_task);
 #endif
-    team->t.t_invoke     = invoker;  /* TODO move this to root, maybe */
+    KMP_CHECK_UPDATE(team->t.t_invoke, invoker);  /* TODO move this to root, maybe */
     // TODO: parent_team->t.t_level == INT_MAX ???
 #if OMP_40_ENABLED
     if ( !master_th->th.th_teams_microtask || level > teams_level ) {
 #endif /* OMP_40_ENABLED */
-        team->t.t_level        = parent_team->t.t_level + 1;
-        team->t.t_active_level = parent_team->t.t_active_level + 1;
+        int new_level = parent_team->t.t_level + 1;
+        KMP_CHECK_UPDATE(team->t.t_level, new_level);
+        new_level = parent_team->t.t_active_level + 1;
+        KMP_CHECK_UPDATE(team->t.t_active_level, new_level);
 #if OMP_40_ENABLED
     } else {
         // AC: Do not increase parallel level at start of the teams construct
-        team->t.t_level        = parent_team->t.t_level;
-        team->t.t_active_level = parent_team->t.t_active_level;
+        int new_level = parent_team->t.t_level;
+        KMP_CHECK_UPDATE(team->t.t_level, new_level);
+        new_level = parent_team->t.t_active_level;
+        KMP_CHECK_UPDATE(team->t.t_active_level, new_level);
     }
 #endif /* OMP_40_ENABLED */
-    team->t.t_sched      = get__sched_2(parent_team, master_tid); // set master's schedule as new run-time schedule
+    kmp_r_sched_t new_sched = get__sched_2(parent_team, master_tid);
+    if (team->t.t_sched.r_sched_type != new_sched.r_sched_type || new_sched.chunk != new_sched.chunk)
+        team->t.t_sched = new_sched; // set master's schedule as new run-time schedule
 
 #if OMP_40_ENABLED
-    team->t.t_cancel_request = cancel_noreq;
+    KMP_CHECK_UPDATE(team->t.t_cancel_request, cancel_noreq);
 #endif
 
     // Update the floating point rounding in the team if required.
     propagateFPControl(team);
 
     if ( __kmp_tasking_mode != tskm_immediate_exec ) {
         // Set master's task team to team's task team. Unless this is hot team, it should be NULL.
 #if 0
KMP_DEBUG_ASSERT( team != root->r.r_hot_team ||
                         ( team->t.t_parent == root->r.r_root_team || team->t.t_parent->t.t_serialized ) ));
     KMP_MB();
 
     /* now, setup the arguments */
     argv = (void**)team->t.t_argv;
 #if OMP_40_ENABLED
     if ( ap ) {
 #endif /* OMP_40_ENABLED */
-        for ( i=argc-1; i >= 0; --i )
+        for ( i=argc-1; i >= 0; --i ) {
 // TODO: revert workaround for Intel(R) 64 tracker #96
 #if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
-            *argv++ = va_arg( *ap, void * );
+            void *new_argv = va_arg(*ap, void *);
 #else
-            *argv++ = va_arg( ap, void * );
+            void *new_argv = va_arg(ap, void *);
 #endif
+            KMP_CHECK_UPDATE(*argv, new_argv);
+            argv++;
+        }
 #if OMP_40_ENABLED
     } else {
-        for ( i=0; i < argc; ++i )
+        for ( i=0; i < argc; ++i ) {
             // Get args from parent team for teams construct
-            argv[i] = team->t.t_parent->t.t_argv[i];
+            KMP_CHECK_UPDATE(argv[i], team->t.t_parent->t.t_argv[i]);
+        }
     }
 #endif /* OMP_40_ENABLED */
 
     /* now actually fork the threads */
-    team->t.t_master_active = master_active;
+    KMP_CHECK_UPDATE(team->t.t_master_active, master_active);
     if (!root->r.r_active) // Only do assignment if it prevents cache ping-pong
         root->r.r_active = TRUE;
 
     __kmp_fork_team_threads( root, team, master_th, gtid );
     __kmp_setup_icv_copy( team, nthreads, &master_th->th.th_current_task->td_icvs, loc );
 
 #if OMPT_SUPPORT
     master_th->th.ompt_thread_info.state = ompt_state_work_parallel;

  * IF YOU TOUCH THIS ROUTINE, RUN EPCC C SYNCBENCH ON A BIG-IRON MACHINE!!!
  */
 static void
 __kmp_reinitialize_team( kmp_team_t *team, kmp_internal_control_t *new_icvs, ident_t *loc ) {
     KF_TRACE( 10, ( "__kmp_reinitialize_team: enter this_thread=%p team=%p\n",
                     team->t.t_threads[0], team ) );
     KMP_DEBUG_ASSERT( team && new_icvs);
     KMP_DEBUG_ASSERT( ( ! TCR_4(__kmp_init_parallel) ) || new_icvs->nproc );
-    team->t.t_ident = loc;
+    KMP_CHECK_UPDATE(team->t.t_ident, loc);
 
-    team->t.t_id = KMP_GEN_TEAM_ID();
+    KMP_CHECK_UPDATE(team->t.t_id, KMP_GEN_TEAM_ID());
 
     // Copy ICVs to the master thread's implicit taskdata
     __kmp_init_implicit_task( loc, team->t.t_threads[0], team, 0, FALSE );
     copy_icvs(&team->t.t_implicit_task_taskdata[0].td_icvs, new_icvs);
 
     KF_TRACE( 10, ( "__kmp_reinitialize_team: exit this_thread=%p team=%p\n",
                     team->t.t_threads[0], team ) );
 }
#endif
            put that case first. */
         if (team->t.t_nproc == new_nproc) { // Check changes in number of threads
             KA_TRACE( 20, ("__kmp_allocate_team: reusing hot team\n" ));
             // This case can mean that omp_set_num_threads() was called and the hot team size
             // was already reduced, so we check the special flag
             if ( team->t.t_size_changed == -1 ) {
                 team->t.t_size_changed = 1;
             } else {
-                team->t.t_size_changed = 0;
+                KMP_CHECK_UPDATE(team->t.t_size_changed, 0);
             }
 
             // TODO???: team->t.t_max_active_levels = new_max_active_levels;
-            team->t.t_sched =  new_icvs->sched;
+            kmp_r_sched_t new_sched = new_icvs->sched;
+            if (team->t.t_sched.r_sched_type != new_sched.r_sched_type || new_sched.chunk != new_sched.chunk)
+                team->t.t_sched = new_sched; // set master's schedule as new run-time schedule
 
             __kmp_reinitialize_team( team, new_icvs, root->r.r_uber_thread->th.th_ident );
 
             KF_TRACE( 10, ("__kmp_allocate_team2: T#%d, this_thread=%p team=%p\n",
                            0, team->t.t_threads[0], team ) );
             __kmp_push_current_task_to_thread( team->t.t_threads[ 0 ], team, 0 );
 
 #if OMP_40_ENABLED
 # if KMP_AFFINITY_SUPPORTED
             if ( ( team->t.t_size_changed == 0 )
               && ( team->t.t_proc_bind == new_proc_bind ) ) {
                 KA_TRACE( 200, ("__kmp_allocate_team: reusing hot team #%d bindings: proc_bind = %d, partition = [%d,%d]\n",
                   team->t.t_id, new_proc_bind, team->t.t_first_place,
                   team->t.t_last_place ) );
             }
             else {
-                team->t.t_proc_bind = new_proc_bind;
+                KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
                 __kmp_partition_places( team );
             }
 # else
             if ( team->t.t_proc_bind != new_proc_bind ) {
                 team->t.t_proc_bind = new_proc_bind;
             }
 # endif /* KMP_AFFINITY_SUPPORTED */
 #endif /* OMP_40_ENABLED */

 #endif
                 }
             }
         }
 #endif // KMP_NESTED_HOT_TEAMS
 
         /* reallocate space for arguments if necessary */
         __kmp_alloc_argv_entries( argc, team, TRUE );
-        team->t.t_argc     = argc;
+        KMP_CHECK_UPDATE(team->t.t_argc, argc);
         //
         // The hot team re-uses the previous task team,
         // if untouched during the previous release->gather phase.
         //
 
         KF_TRACE( 10, ( " hot_team = %p\n", team ) );
 
 #if KMP_DEBUG
for( team = (kmp_team_t*) __kmp_team_pool ; (team) ; )
 
             KA_TRACE( 20, ( "__kmp_allocate_team: setting task_team[0] %p and task_team[1] %p to NULL\n",
                             &team->t.t_task_team[0], &team->t.t_task_team[1]) );
             team->t.t_task_team[0] = NULL;
             team->t.t_task_team[1] = NULL;
 
             /* reallocate space for arguments if necessary */
             __kmp_alloc_argv_entries( argc, team, TRUE );
-            team->t.t_argc     = argc;
+            KMP_CHECK_UPDATE(team->t.t_argc, argc);
 
             KA_TRACE( 20, ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n",
                             team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE ));
             { // Initialize barrier data.
                 int b;
                 for ( b = 0; b < bs_last_barrier; ++ b) {
                     team->t.t_bar[ b ].b_arrived        = KMP_INIT_BARRIER_STATE;
 #if USE_DEBUGGER