Fix hanging barriers if number of parallel regions exceeds UINT_MAX

[jlpeyton]

Summary

The fix is to make b_arrived flag 64 bit in both structures - kmp_balign_team_t and kmp_balign_t. Otherwise when flag in kmp_balign_team_t wrapped over UINT_MAX the library hangs.

[AndreyChurbanov]

LGTM

openmp/trunk/runtime/src/kmp.h

 
 typedef union kmp_barrier_union kmp_balign_t;
 
 /* Team barrier needs only non-volatile arrived counter */
 union KMP_ALIGN_CACHE kmp_barrier_team_union {
     double       b_align;        /* use worst case alignment */
     char         b_pad[ CACHE_LINE ];
     struct {
-        kmp_uint     b_arrived;       /* STATE => task reached synch point. */
+        kmp_uint64   b_arrived;       /* STATE => task reached synch point. */
 #if USE_DEBUGGER
         // The following two fields are indended for the debugger solely. Only master of the team accesses
         // these fields: the first one is increased by 1 when master arrives to a barrier, the
         // second one is increased by one when all the threads arrived.
         kmp_uint     b_master_arrived;
         kmp_uint     b_team_arrived;
 #endif
     };

openmp/trunk/runtime/src/kmp_barrier.cpp

#if USE_ITT_BUILD && USE_ITT_NOTIFY
     // Barrier imbalance - save arrive time to the thread
     if(__kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 2) {
         this_thr->th.th_bar_arrive_time = this_thr->th.th_bar_min_time = __itt_get_timestamp();
     }
 #endif
     // We now perform a linear reduction to signal that all of the threads have arrived.
     if (!KMP_MASTER_TID(tid)) {
         KA_TRACE(20, ("__kmp_linear_barrier_gather: T#%d(%d:%d) releasing T#%d(%d:%d)"
-                      "arrived(%p): %u => %u\n", gtid, team->t.t_id, tid,
+                      "arrived(%p): %llu => %llu\n", gtid, team->t.t_id, tid,
                       __kmp_gtid_from_tid(0, team), team->t.t_id, 0, &thr_bar->b_arrived,
                       thr_bar->b_arrived, thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
         // Mark arrival to master thread
         /* After performing this write, a worker thread may not assume that the team is valid
            any more - it could be deallocated by the master thread at any time. */
         kmp_flag_64 flag(&thr_bar->b_arrived, other_threads[0]);
         flag.release();
     } else {
         register kmp_balign_team_t *team_bar = &team->t.t_bar[bt];
         register int nproc = this_thr->th.th_team_nproc;
         register int i;
         // Don't have to worry about sleep bit here or atomic since team setting
-        register kmp_uint new_state = team_bar->b_arrived + KMP_BARRIER_STATE_BUMP;
+        register kmp_uint64 new_state = team_bar->b_arrived + KMP_BARRIER_STATE_BUMP;
 
         // Collect all the worker team member threads.
         for (i=1; i<nproc; ++i) {
 #if KMP_CACHE_MANAGE
             // Prefetch next thread's arrived count
             if (i+1 < nproc)
                 KMP_CACHE_PREFETCH(&other_threads[i+1]->th.th_bar[bt].bb.b_arrived);
 #endif /* KMP_CACHE_MANAGE */
             KA_TRACE(20, ("__kmp_linear_barrier_gather: T#%d(%d:%d) wait T#%d(%d:%d) "
-                          "arrived(%p) == %u\n", gtid, team->t.t_id, tid,
+                          "arrived(%p) == %llu\n", gtid, team->t.t_id, tid,
                             __kmp_gtid_from_tid(i, team), team->t.t_id, i,
                             &other_threads[i]->th.th_bar[bt].bb.b_arrived, new_state));
 
             // Wait for worker thread to arrive
             kmp_flag_64 flag(&other_threads[i]->th.th_bar[bt].bb.b_arrived, new_state);
             flag.wait(this_thr, FALSE
                       USE_ITT_BUILD_ARG(itt_sync_obj) );
 #if USE_ITT_BUILD && USE_ITT_NOTIFY
             // Barrier imbalance - write min of the thread time and the other thread time to the thread.
             if (__kmp_forkjoin_frames_mode == 2) {
                 this_thr->th.th_bar_min_time = KMP_MIN(this_thr->th.th_bar_min_time,
                                                           other_threads[i]->th.th_bar_min_time);
             }
 #endif
             if (reduce) {
                 KA_TRACE(100, ("__kmp_linear_barrier_gather: T#%d(%d:%d) += T#%d(%d:%d)\n", gtid,
                                team->t.t_id, tid, __kmp_gtid_from_tid(i, team), team->t.t_id, i));
                 (*reduce)(this_thr->th.th_local.reduce_data,
                           other_threads[i]->th.th_local.reduce_data);
             }
         }
         // Don't have to worry about sleep bit here or atomic since team setting
         team_bar->b_arrived = new_state;
-        KA_TRACE(20, ("__kmp_linear_barrier_gather: T#%d(%d:%d) set team %d arrived(%p) = %u\n",
+        KA_TRACE(20, ("__kmp_linear_barrier_gather: T#%d(%d:%d) set team %d arrived(%p) = %llu\n",
                       gtid, team->t.t_id, tid, team->t.t_id, &team_bar->b_arrived, new_state));
     }
     KA_TRACE(20, ("__kmp_linear_barrier_gather: T#%d(%d:%d) exit for barrier type %d\n",
                   gtid, team->t.t_id, tid, bt));
 }
 
 static void
 __kmp_linear_barrier_release(enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
__kmp_tree_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
     register kmp_team_t *team = this_thr->th.th_team;
     register kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
     register kmp_info_t **other_threads = team->t.t_threads;
     register kmp_uint32 nproc = this_thr->th.th_team_nproc;
     register kmp_uint32 branch_bits = __kmp_barrier_gather_branch_bits[bt];
     register kmp_uint32 branch_factor = 1 << branch_bits;
     register kmp_uint32 child;
     register kmp_uint32 child_tid;
-    register kmp_uint new_state;
+    register kmp_uint64 new_state;
 
     KA_TRACE(20, ("__kmp_tree_barrier_gather: T#%d(%d:%d) enter for barrier type %d\n",
                   gtid, team->t.t_id, tid, bt));
     KMP_DEBUG_ASSERT(this_thr == other_threads[this_thr->th.th_info.ds.ds_tid]);
 
 #if USE_ITT_BUILD && USE_ITT_NOTIFY
     // Barrier imbalance - save arrive time to the thread
     if(__kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 2) {
if (child_tid < nproc) {
             register kmp_info_t *child_thr = other_threads[child_tid];
             register kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb;
 #if KMP_CACHE_MANAGE
             // Prefetch next thread's arrived count
             if (child+1 <= branch_factor && child_tid+1 < nproc)
                 KMP_CACHE_PREFETCH(&other_threads[child_tid+1]->th.th_bar[bt].bb.b_arrived);
 #endif /* KMP_CACHE_MANAGE */
             KA_TRACE(20, ("__kmp_tree_barrier_gather: T#%d(%d:%d) wait T#%d(%d:%u) "
-                          "arrived(%p) == %u\n", gtid, team->t.t_id, tid,
+                          "arrived(%p) == %llu\n", gtid, team->t.t_id, tid,
                             __kmp_gtid_from_tid(child_tid, team), team->t.t_id, child_tid,
                             &child_bar->b_arrived, new_state));
             // Wait for child to arrive
             kmp_flag_64 flag(&child_bar->b_arrived, new_state);
             flag.wait(this_thr, FALSE
                       USE_ITT_BUILD_ARG(itt_sync_obj) );
 #if USE_ITT_BUILD && USE_ITT_NOTIFY
             // Barrier imbalance - write min of the thread time and a child time to the thread.
#endif
         }
         while (child <= branch_factor && child_tid < nproc);
     }
 
     if (!KMP_MASTER_TID(tid)) { // Worker threads
         register kmp_int32 parent_tid = (tid - 1) >> branch_bits;
 
         KA_TRACE(20, ("__kmp_tree_barrier_gather: T#%d(%d:%d) releasing T#%d(%d:%d) "
-                      "arrived(%p): %u => %u\n", gtid, team->t.t_id, tid,
+                      "arrived(%p): %llu => %llu\n", gtid, team->t.t_id, tid,
                       __kmp_gtid_from_tid(parent_tid, team), team->t.t_id, parent_tid,
                       &thr_bar->b_arrived, thr_bar->b_arrived,
                       thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
 
         // Mark arrival to parent thread
         /* After performing this write, a worker thread may not assume that the team is valid
            any more - it could be deallocated by the master thread at any time.  */
         kmp_flag_64 flag(&thr_bar->b_arrived, other_threads[parent_tid]);
         flag.release();
     } else {
         // Need to update the team arrived pointer if we are the master thread
         if (nproc > 1) // New value was already computed above
             team->t.t_bar[bt].b_arrived = new_state;
         else
             team->t.t_bar[bt].b_arrived += KMP_BARRIER_STATE_BUMP;
-        KA_TRACE(20, ("__kmp_tree_barrier_gather: T#%d(%d:%d) set team %d arrived(%p) = %u\n",
+        KA_TRACE(20, ("__kmp_tree_barrier_gather: T#%d(%d:%d) set team %d arrived(%p) = %llu\n",
                       gtid, team->t.t_id, tid, team->t.t_id,
                       &team->t.t_bar[bt].b_arrived, team->t.t_bar[bt].b_arrived));
     }
     KA_TRACE(20, ("__kmp_tree_barrier_gather: T#%d(%d:%d) exit for barrier type %d\n",
                   gtid, team->t.t_id, tid, bt));
 }
 
 static void

 __kmp_hyper_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid,
                            void (*reduce)(void *, void *)
                            USE_ITT_BUILD_ARG(void *itt_sync_obj) )
 {
     KMP_TIME_DEVELOPER_BLOCK(KMP_hyper_gather);
     register kmp_team_t *team = this_thr->th.th_team;
     register kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
     register kmp_info_t **other_threads = team->t.t_threads;
-    register kmp_uint new_state = KMP_BARRIER_UNUSED_STATE;
+    register kmp_uint64 new_state = KMP_BARRIER_UNUSED_STATE;
     register kmp_uint32 num_threads = this_thr->th.th_team_nproc;
     register kmp_uint32 branch_bits = __kmp_barrier_gather_branch_bits[bt];
     register kmp_uint32 branch_factor = 1 << branch_bits;
     register kmp_uint32 offset;
     register kmp_uint32 level;
 
     KA_TRACE(20, ("__kmp_hyper_barrier_gather: T#%d(%d:%d) enter for barrier type %d\n",
                   gtid, team->t.t_id, tid, bt));
#endif
     {
         register kmp_uint32 child;
         register kmp_uint32 child_tid;
 
         if (((tid >> level) & (branch_factor - 1)) != 0) {
             register kmp_int32 parent_tid = tid & ~((1 << (level + branch_bits)) -1);
 
             KA_TRACE(20, ("__kmp_hyper_barrier_gather: T#%d(%d:%d) releasing T#%d(%d:%d) "
-                          "arrived(%p): %u => %u\n", gtid, team->t.t_id, tid,
+                          "arrived(%p): %llu => %llu\n", gtid, team->t.t_id, tid,
                           __kmp_gtid_from_tid(parent_tid, team), team->t.t_id, parent_tid,
                           &thr_bar->b_arrived, thr_bar->b_arrived,
                           thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
             // Mark arrival to parent thread
             /* After performing this write (in the last iteration of the enclosing for loop),
                a worker thread may not assume that the team is valid any more - it could be
                deallocated by the master thread at any time.  */
             p_flag.set_waiter(other_threads[parent_tid]);
for (level=0, offset=1; offset<num_threads; level+=branch_bits, offset<<=branch_bits)
             register kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb;
 #if KMP_CACHE_MANAGE
             register kmp_uint32 next_child_tid = child_tid + (1 << level);
             // Prefetch next thread's arrived count
             if (child+1 < branch_factor && next_child_tid < num_threads)
                 KMP_CACHE_PREFETCH(&other_threads[next_child_tid]->th.th_bar[bt].bb.b_arrived);
 #endif /* KMP_CACHE_MANAGE */
             KA_TRACE(20, ("__kmp_hyper_barrier_gather: T#%d(%d:%d) wait T#%d(%d:%u) "
-                          "arrived(%p) == %u\n", gtid, team->t.t_id, tid,
+                          "arrived(%p) == %llu\n", gtid, team->t.t_id, tid,
                           __kmp_gtid_from_tid(child_tid, team), team->t.t_id, child_tid,
                           &child_bar->b_arrived, new_state));
             // Wait for child to arrive
             kmp_flag_64 c_flag(&child_bar->b_arrived, new_state);
             c_flag.wait(this_thr, FALSE
                         USE_ITT_BUILD_ARG(itt_sync_obj) );
 #if USE_ITT_BUILD && USE_ITT_NOTIFY
             // Barrier imbalance - write min of the thread time and a child time to the thread.
#endif
     }
 
     if (KMP_MASTER_TID(tid)) {
         // Need to update the team arrived pointer if we are the master thread
         if (new_state == KMP_BARRIER_UNUSED_STATE)
             team->t.t_bar[bt].b_arrived += KMP_BARRIER_STATE_BUMP;
         else
             team->t.t_bar[bt].b_arrived = new_state;
-        KA_TRACE(20, ("__kmp_hyper_barrier_gather: T#%d(%d:%d) set team %d arrived(%p) = %u\n",
+        KA_TRACE(20, ("__kmp_hyper_barrier_gather: T#%d(%d:%d) set team %d arrived(%p) = %llu\n",
                       gtid, team->t.t_id, tid, team->t.t_id,
                       &team->t.t_bar[bt].b_arrived, team->t.t_bar[bt].b_arrived));
     }
     KA_TRACE(20, ("__kmp_hyper_barrier_gather: T#%d(%d:%d) exit for barrier type %d\n",
                   gtid, team->t.t_id, tid, bt));
 }
 
 // The reverse versions seem to beat the forward versions overall
#endif
 
     (void)__kmp_init_hierarchical_barrier_thread(bt, thr_bar, nproc, gtid, tid, team);
 
     if (thr_bar->my_level) { // not a leaf (my_level==0 means leaf)
         register kmp_int32 child_tid;
         new_state = (kmp_uint64)team->t.t_bar[bt].b_arrived + KMP_BARRIER_STATE_BUMP;
         if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME && thr_bar->use_oncore_barrier) {
             if (thr_bar->leaf_kids) { // First, wait for leaf children to check-in on my b_arrived flag
-                kmp_uint64 leaf_state = KMP_MASTER_TID(tid) ? thr_bar->b_arrived | thr_bar->leaf_state : (kmp_uint64)team->t.t_bar[bt].b_arrived | thr_bar->leaf_state;
+                kmp_uint64 leaf_state = KMP_MASTER_TID(tid) ? thr_bar->b_arrived | thr_bar->leaf_state : team->t.t_bar[bt].b_arrived | thr_bar->leaf_state;
                 kmp_flag_64 flag(&thr_bar->b_arrived, leaf_state);
                 flag.wait(this_thr, FALSE
                           USE_ITT_BUILD_ARG(itt_sync_obj) );
                 if (reduce) {
                     for (child_tid=tid+1; child_tid<=tid+thr_bar->leaf_kids; ++child_tid) {
                         KA_TRACE(100, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) += T#%d(%d:%d)\n",
                                        gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
                                        team->t.t_id, child_tid));
                         (*reduce)(this_thr->th.th_local.reduce_data, other_threads[child_tid]->th.th_local.reduce_data);
                     }
                 }
                 (void) KMP_TEST_THEN_AND64((volatile kmp_int64 *)&thr_bar->b_arrived, ~(thr_bar->leaf_state)); // clear leaf_state bits
             }
             // Next, wait for higher level children on each child's b_arrived flag
             for (kmp_uint32 d=1; d<thr_bar->my_level; ++d) { // gather lowest level threads first, but skip 0
                 kmp_uint32 last = tid+thr_bar->skip_per_level[d+1], skip = thr_bar->skip_per_level[d];
                 if (last > nproc) last = nproc;
                 for (child_tid=tid+skip; child_tid<(int)last; child_tid+=skip) {
                     register kmp_info_t *child_thr = other_threads[child_tid];
                     register kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb;
                     KA_TRACE(20, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) wait T#%d(%d:%d) "
-                                  "arrived(%p) == %u\n",
+                                  "arrived(%p) == %llu\n",
                                   gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
                                   team->t.t_id, child_tid, &child_bar->b_arrived, new_state));
                     kmp_flag_64 flag(&child_bar->b_arrived, new_state);
                     flag.wait(this_thr, FALSE
                               USE_ITT_BUILD_ARG(itt_sync_obj) );
                     if (reduce) {
                         KA_TRACE(100, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) += T#%d(%d:%d)\n",
                                        gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
                                        team->t.t_id, child_tid));
                         (*reduce)(this_thr->th.th_local.reduce_data, child_thr->th.th_local.reduce_data);
                     }
                 }
             }
         }
         else { // Blocktime is not infinite
             for (kmp_uint32 d=0; d<thr_bar->my_level; ++d) { // Gather lowest level threads first
                 kmp_uint32 last = tid+thr_bar->skip_per_level[d+1], skip = thr_bar->skip_per_level[d];
                 if (last > nproc) last = nproc;
                 for (child_tid=tid+skip; child_tid<(int)last; child_tid+=skip) {
                     register kmp_info_t *child_thr = other_threads[child_tid];
                     register kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb;
                     KA_TRACE(20, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) wait T#%d(%d:%d) "
-                                  "arrived(%p) == %u\n",
+                                  "arrived(%p) == %llu\n",
                                   gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
                                   team->t.t_id, child_tid, &child_bar->b_arrived, new_state));
                     kmp_flag_64 flag(&child_bar->b_arrived, new_state);
                     flag.wait(this_thr, FALSE
                               USE_ITT_BUILD_ARG(itt_sync_obj) );
                     if (reduce) {
                         KA_TRACE(100, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) += T#%d(%d:%d)\n",
                                        gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
                                        team->t.t_id, child_tid));
                         (*reduce)(this_thr->th.th_local.reduce_data, child_thr->th.th_local.reduce_data);
                     }
                 }
             }
         }
     }
     // All subordinates are gathered; now release parent if not master thread
 
     if (!KMP_MASTER_TID(tid)) { // worker threads release parent in hierarchy
         KA_TRACE(20, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) releasing T#%d(%d:%d) "
-                      "arrived(%p): %u => %u\n", gtid, team->t.t_id, tid,
+                      "arrived(%p): %llu => %llu\n", gtid, team->t.t_id, tid,
                       __kmp_gtid_from_tid(thr_bar->parent_tid, team), team->t.t_id, thr_bar->parent_tid,
                       &thr_bar->b_arrived, thr_bar->b_arrived, thr_bar->b_arrived+KMP_BARRIER_STATE_BUMP));
         /* Mark arrival to parent: After performing this write, a worker thread may not assume that
            the team is valid any more - it could be deallocated by the master thread at any time. */
         if (thr_bar->my_level || __kmp_dflt_blocktime != KMP_MAX_BLOCKTIME
             || !thr_bar->use_oncore_barrier) { // Parent is waiting on my b_arrived flag; release it
             kmp_flag_64 flag(&thr_bar->b_arrived, other_threads[thr_bar->parent_tid]);
             flag.release();
         }
         else { // Leaf does special release on the "offset" bits of parent's b_arrived flag
-            thr_bar->b_arrived = (kmp_uint64)team->t.t_bar[bt].b_arrived + KMP_BARRIER_STATE_BUMP;
+            thr_bar->b_arrived = team->t.t_bar[bt].b_arrived + KMP_BARRIER_STATE_BUMP;
             kmp_flag_oncore flag(&thr_bar->parent_bar->b_arrived, thr_bar->offset);
             flag.set_waiter(other_threads[thr_bar->parent_tid]);
             flag.release();
         }
     } else { // Master thread needs to update the team's b_arrived value
-        team->t.t_bar[bt].b_arrived = (kmp_uint32)new_state;
-        KA_TRACE(20, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) set team %d arrived(%p) = %u\n",
+        team->t.t_bar[bt].b_arrived = new_state;
+        KA_TRACE(20, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) set team %d arrived(%p) = %llu\n",
                       gtid, team->t.t_id, tid, team->t.t_id, &team->t.t_bar[bt].b_arrived, team->t.t_bar[bt].b_arrived));
     }
     // Is the team access below unsafe or just technically invalid?
     KA_TRACE(20, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) exit for barrier type %d\n",
                   gtid, team->t.t_id, tid, bt));
 }
 
 static void

openmp/trunk/runtime/src/kmp_itt.inl

#if USE_ITT_NOTIFY
     // If the function is called from __kmp_fork_barrier, team pointer can be NULL. This "if"
     // helps to avoid crash. However, this is not complete solution, and reporting fork/join
     // barriers to ITT should be revisited.
 
     if ( team != NULL ) {
 
         // Master thread increases b_arrived by KMP_BARRIER_STATE_BUMP each time. Divide b_arrived
         // by KMP_BARRIER_STATE_BUMP to get plain barrier counter.
-        kmp_uint counter = team->t.t_bar[ bt ].b_arrived / KMP_BARRIER_STATE_BUMP + delta;
+        kmp_uint64 counter = team->t.t_bar[ bt ].b_arrived / KMP_BARRIER_STATE_BUMP + delta;
         // Now form the barrier id. Encode barrier type (bt) in barrier id too, so barriers of
         // different types do not have the same ids.
         KMP_BUILD_ASSERT( sizeof( kmp_team_t ) >= bs_last_barrier );
             // This conditon is a must (we would have zero divide otherwise).
         KMP_BUILD_ASSERT( sizeof( kmp_team_t ) >= 2 * bs_last_barrier );
             // More strong condition: make sure we have room at least for for two differtent ids
             // (for each barrier type).
         object =
             reinterpret_cast< void * >(
                 kmp_uintptr_t( team )
                     + counter % ( sizeof( kmp_team_t ) / bs_last_barrier ) * bs_last_barrier
                     + bt
             );
         KMP_ITT_DEBUG_LOCK();
-        KMP_ITT_DEBUG_PRINT( "[bar obj] type=%d, counter=%d, object=%p\n", bt, counter, object );
+        KMP_ITT_DEBUG_PRINT( "[bar obj] type=%d, counter=%lld, object=%p\n", bt, counter, object );
 
         if ( set_name ) {
             ident_t const * loc  = NULL;
             char const *    src  = NULL;
             char const *    type = "OMP Barrier";
             switch ( bt ) {
                 case bs_plain_barrier : {
                     // For plain barrier compiler calls __kmpc_barrier() function, which saves

openmp/trunk/runtime/src/kmp_runtime.c

if ( !use_hot_team ) {
         for ( i=1 ;  i < team->t.t_nproc ; i++ ) {
 
             /* fork or reallocate a new thread and install it in team */
             kmp_info_t *thr = __kmp_allocate_thread( root, team, i );
             team->t.t_threads[ i ] = thr;
             KMP_DEBUG_ASSERT( thr );
             KMP_DEBUG_ASSERT( thr->th.th_team == team );
             /* align team and thread arrived states */
-            KA_TRACE( 20, ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived T#%d(%d:%d) join =%u, plain=%u\n",
+            KA_TRACE( 20, ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived T#%d(%d:%d) join =%llu, plain=%llu\n",
                             __kmp_gtid_from_tid( 0, team ), team->t.t_id, 0,
                             __kmp_gtid_from_tid( i, team ), team->t.t_id, i,
                             team->t.t_bar[ bs_forkjoin_barrier ].b_arrived,
                             team->t.t_bar[ bs_plain_barrier ].b_arrived ) );
 #if OMP_40_ENABLED
             thr->th.th_teams_microtask = master_th->th.th_teams_microtask;
             thr->th.th_teams_level     = master_th->th.th_teams_level;
             thr->th.th_teams_size      = master_th->th.th_teams_size;

 #endif
 
             /* allocate new threads for the hot team */
             for( f = team->t.t_nproc  ;  f < new_nproc  ;  f++ ) {
                 kmp_info_t * new_worker = __kmp_allocate_thread( root, team, f );
                 KMP_DEBUG_ASSERT( new_worker );
                 team->t.t_threads[ f ] = new_worker;
 
-                KA_TRACE( 20, ("__kmp_allocate_team: team %d init T#%d arrived: join=%u, plain=%u\n",
+                KA_TRACE( 20, ("__kmp_allocate_team: team %d init T#%d arrived: join=%llu, plain=%llu\n",
                                 team->t.t_id, __kmp_gtid_from_tid( f, team ), team->t.t_id, f,
                                 team->t.t_bar[bs_forkjoin_barrier].b_arrived,
                                 team->t.t_bar[bs_plain_barrier].b_arrived ) );
 
                 { // Initialize barrier data for new threads.
                     int b;
                     kmp_balign_t * balign = new_worker->th.th_bar;
                     for( b = 0; b < bs_last_barrier; ++ b ) {