Performance improvement: favor accessing thread struct as opposed to team struct

[tlwilmar]

Summary

Replaced readings of nproc from team structure with ones from
thread structure, to improve performance.

Patch by Andrey Churbanov.

[jlpeyton]

LGTM

openmp/trunk/runtime/src/kmp_dispatch.cpp

#endif
                     traits_t< ST >::spec );
                 KD_TRACE(10, ( buff, gtid, schedule, chunk ) );
                 __kmp_str_free( &buff );
             }
             #endif
         }
 
         /* guided analytical not safe for too many threads */
-        if ( team->t.t_nproc > 1<<20 && schedule == kmp_sch_guided_analytical_chunked ) {
+        if ( schedule == kmp_sch_guided_analytical_chunked && th->th.th_team_nproc > 1<<20 ) {
             schedule = kmp_sch_guided_iterative_chunked;
             KMP_WARNING( DispatchManyThreads );
         }
         pr->u.p.parm1 = chunk;
     }
     KMP_ASSERT2( (kmp_sch_lower < schedule && schedule < kmp_sch_upper),
                 "unknown scheduling type" );
 
if ( __kmp_env_consistency_check ) {
             pr->pushed_ws = ct_none;
         }
     }
 
     switch ( schedule ) {
     #if  ( KMP_STATIC_STEAL_ENABLED && KMP_ARCH_X86_64 )
     case kmp_sch_static_steal:
         {
-            T nproc = team->t.t_nproc;
+            T nproc = th->th.th_team_nproc;
             T ntc, init;
 
             KD_TRACE(100, ("__kmp_dispatch_init: T#%d kmp_sch_static_steal case\n", gtid ) );
 
             ntc = (tc % chunk ? 1 : 0) + tc / chunk;
             if ( nproc > 1 && ntc >= nproc ) {
                 T id = __kmp_tid_from_gtid(gtid);
                 T small_chunk, extras;
case kmp_sch_static_steal:
                 schedule = kmp_sch_static_balanced;
                 /* too few iterations: fall-through to kmp_sch_static_balanced */
             } // if
             /* FALL-THROUGH to static balanced */
         } // case
     #endif
     case kmp_sch_static_balanced:
         {
-            T nproc = team->t.t_nproc;
+            T nproc = th->th.th_team_nproc;
             T init, limit;
 
             KD_TRACE(100, ("__kmp_dispatch_init: T#%d kmp_sch_static_balanced case\n",
                             gtid ) );
 
             if ( nproc > 1 ) {
                 T id = __kmp_tid_from_gtid(gtid);
 
#endif
             if ( pr->ordered ) {
                 pr->u.p.ordered_lower = init;
                 pr->u.p.ordered_upper = limit;
             }
             break;
         } // case
     case kmp_sch_guided_iterative_chunked :
         {
-            T nproc = team->t.t_nproc;
+            T nproc = th->th.th_team_nproc;
             KD_TRACE(100,("__kmp_dispatch_init: T#%d kmp_sch_guided_iterative_chunked case\n",gtid));
 
             if ( nproc > 1 ) {
                 if ( (2L * chunk + 1 ) * nproc >= tc ) {
                     /* chunk size too large, switch to dynamic */
                     schedule = kmp_sch_dynamic_chunked;
                 } else {
                     // when remaining iters become less than parm2 - switch to dynamic
                     pr->u.p.parm2 = guided_int_param * nproc * ( chunk + 1 );
                     *(double*)&pr->u.p.parm3 = guided_flt_param / nproc;   // may occupy parm3 and parm4
                 }
             } else {
                 KD_TRACE(100,("__kmp_dispatch_init: T#%d falling-through to kmp_sch_static_greedy\n",gtid));
                 schedule = kmp_sch_static_greedy;
                 /* team->t.t_nproc == 1: fall-through to kmp_sch_static_greedy */
                 KD_TRACE(100,("__kmp_dispatch_init: T#%d kmp_sch_static_greedy case\n",gtid));
                 pr->u.p.parm1 = tc;
             } // if
         } // case
         break;
     case kmp_sch_guided_analytical_chunked:
         {
-            T nproc = team->t.t_nproc;
+            T nproc = th->th.th_team_nproc;
             KD_TRACE(100, ("__kmp_dispatch_init: T#%d kmp_sch_guided_analytical_chunked case\n", gtid));
 
             if ( nproc > 1 ) {
                 if ( (2L * chunk + 1 ) * nproc >= tc ) {
                     /* chunk size too large, switch to dynamic */
                     schedule = kmp_sch_dynamic_chunked;
                 } else {
                     /* commonly used term: (2 nproc - 1)/(2 nproc) */
case kmp_sch_guided_analytical_chunked:
                 schedule = kmp_sch_static_greedy;
                 /* team->t.t_nproc == 1: fall-through to kmp_sch_static_greedy */
                 pr->u.p.parm1 = tc;
             } // if
         } // case
         break;
     case kmp_sch_static_greedy:
         KD_TRACE(100,("__kmp_dispatch_init: T#%d kmp_sch_static_greedy case\n",gtid));
-            pr->u.p.parm1 = ( team -> t.t_nproc > 1 ) ?
-                ( tc + team->t.t_nproc - 1 ) / team->t.t_nproc :
+            pr->u.p.parm1 = ( th->th.th_team_nproc > 1 ) ?
+                ( tc + th->th.th_team_nproc - 1 ) / th->th.th_team_nproc :
                 tc;
         break;
     case kmp_sch_static_chunked :
     case kmp_sch_dynamic_chunked :
         if ( pr->u.p.parm1 <= 0 ) {
             pr->u.p.parm1 = KMP_DEFAULT_CHUNK;
         }
         KD_TRACE(100,("__kmp_dispatch_init: T#%d kmp_sch_static_chunked/kmp_sch_dynamic_chunked cases\n", gtid));
         break;
     case kmp_sch_trapezoidal :
         {
             /* TSS: trapezoid self-scheduling, minimum chunk_size = parm1 */
 
             T parm1, parm2, parm3, parm4;
             KD_TRACE(100, ("__kmp_dispatch_init: T#%d kmp_sch_trapezoidal case\n", gtid ) );
 
             parm1 = chunk;
 
             /* F : size of the first cycle */
-            parm2 = ( tc / (2 * team->t.t_nproc) );
+            parm2 = ( tc / (2 * th->th.th_team_nproc) );
 
             if ( parm2 < 1 ) {
                 parm2 = 1;
             }
 
             /* L : size of the last cycle.  Make sure the last cycle
              *     is not larger than the first cycle.
              */
#endif
                         incr  = pr->u.p.st;
                         limit = parm1 + init - 1;
 
                         if ( (last = (limit >= trip)) != 0 )
                             limit = trip;
 
                         if ( p_st != NULL ) *p_st = incr;
 
-                        pr->u.p.count += team->t.t_nproc;
+                        pr->u.p.count += th->th.th_team_nproc;
 
                         if ( incr == 1 ) {
                             *p_lb = start + init;
                             *p_ub = start + limit;
                         }
                         else {
                             *p_lb = start + init * incr;
                             *p_ub = start + limit * incr;
#if KMP_OS_WINDOWS && KMP_ARCH_X86
                     unsigned int oldFpcw;
                     unsigned int fpcwSet = 0;
     #endif
                     KD_TRACE(100, ("__kmp_dispatch_next: T#%d kmp_sch_guided_chunked analytical case\n",
                                    gtid ) );
 
                     trip  = pr->u.p.tc;
 
-                    KMP_DEBUG_ASSERT(team->t.t_nproc > 1);
-                    KMP_DEBUG_ASSERT((2UL * chunkspec + 1) * (UT)team->t.t_nproc < trip);
+                    KMP_DEBUG_ASSERT(th->th.th_team_nproc > 1);
+                    KMP_DEBUG_ASSERT((2UL * chunkspec + 1) * (UT)th->th.th_team_nproc < trip);
 
                     while(1) { /* this while loop is a safeguard against unexpected zero chunk sizes */
                         chunkIdx = test_then_inc_acq< ST >((volatile ST *) & sh->u.s.iteration );
                         if ( chunkIdx >= (UT)pr->u.p.parm2 ) {
                             --trip;
                             /* use dynamic-style scheduling */
                             init = chunkIdx * chunkspec + pr->u.p.count;
                             /* need to verify init > 0 in case of overflow in the above calculation */
#endif
                 buff = __kmp_str_format(
                     "__kmp_dispatch_next: T#%%d increment num_done:%%%s\n",
                     traits_t< UT >::spec );
                 KD_TRACE(100, ( buff, gtid, sh->u.s.num_done ) );
                 __kmp_str_free( &buff );
             }
             #endif
 
-            if ( (ST)num_done == team->t.t_nproc-1 ) {
+            if ( (ST)num_done == th->th.th_team_nproc - 1 ) {
                 /* NOTE: release this buffer to be reused */
 
                 KMP_MB();       /* Flush all pending memory write invalidates.  */
 
                 sh->u.s.num_done = 0;
                 sh->u.s.iteration = 0;
 
                 /* TODO replace with general release procedure? */