Fix for OMP doacross implementation on Power

[Hahnfeld]

Summary

Power has a weak consistency model so we need memory barriers to
make writes (both from runtime and from user code) available for
all threads.

[Hahnfeld]

runtime/src/kmp_csupport.cpp

3813–3823 ↗

(On Diff #123335)

I'm not really sure if these memory barriers are needed, ie I haven't seen the test fail without them. I put them in because I think that all threads need to see the zeroed memory in flags

[Hahnfeld]

ARM should have the same problem AFAIK

[Hahnfeld]

Add KMP_MB() in __kmpc_doacross_post after discussion with @AlexEichenberger.

[AlexEichenberger]

Changes are correct, a memory barrier is needed between allocating/reseting values to zero and publishing the pointer. That way, another thread see the published pointer only once all of the zeroing is complete.

In other places, an "instruction sync" might be sufficient, but the memory barrier is certainly fine too (though possibly a bit more expensive)

openmp/trunk/runtime/src/kmp_csupport.cpp

flags = (kmp_uint32 *)KMP_COMPARE_AND_STORE_RET32(
       (volatile kmp_int32 *)&sh_buf->doacross_flags, NULL, 1);
 #else
   flags = (kmp_uint32 *)KMP_COMPARE_AND_STORE_RET64(
       (volatile kmp_int64 *)&sh_buf->doacross_flags, NULL, 1LL);
 #endif
   if (flags == NULL) {
     // we are the first thread, allocate the array of flags
     size_t size = trace_count / 8 + 8; // in bytes, use single bit per iteration
-    sh_buf->doacross_flags = (kmp_uint32 *)__kmp_thread_calloc(th, size, 1);
+    flags = (kmp_uint32 *)__kmp_thread_calloc(th, size, 1);
+    KMP_MB();
+    sh_buf->doacross_flags = flags;
   } else if (flags == (kmp_uint32 *)1) {
 #if KMP_32_BIT_ARCH
     // initialization is still in progress, need to wait
     while (*(volatile kmp_int32 *)&sh_buf->doacross_flags == 1)
 #else
     while (*(volatile kmp_int64 *)&sh_buf->doacross_flags == 1LL)
 #endif
       KMP_YIELD(TRUE);
+    KMP_MB();
+  } else {
+    KMP_MB();
   }
   KMP_DEBUG_ASSERT(sh_buf->doacross_flags > (kmp_uint32 *)1); // check ptr value
   pr_buf->th_doacross_flags =
       sh_buf->doacross_flags; // save private copy in order to not
   // touch shared buffer on each iteration
   KA_TRACE(20, ("__kmpc_doacross_init() exit: T#%d\n", gtid));
 }
 
for (i = 1; i < num_dims; ++i) {
     iter_number = iter + ln * iter_number;
   }
   shft = iter_number % 32; // use 32-bit granularity
   iter_number >>= 5; // divided by 32
   flag = 1 << shft;
   while ((flag & pr_buf->th_doacross_flags[iter_number]) == 0) {
     KMP_YIELD(TRUE);
   }
+  KMP_MB();
   KA_TRACE(20,
            ("__kmpc_doacross_wait() exit: T#%d wait for iter %lld completed\n",
             gtid, (iter_number << 5) + shft));
 }
 
 void __kmpc_doacross_post(ident_t *loc, int gtid, long long *vec) {
   kmp_int32 shft, num_dims, i;
   kmp_uint32 flag;
for (i = 1; i < num_dims; ++i) {
     } else { // st < 0
       iter = (kmp_uint64)(lo - vec[i]) / (-st);
     }
     iter_number = iter + ln * iter_number;
   }
   shft = iter_number % 32; // use 32-bit granularity
   iter_number >>= 5; // divided by 32
   flag = 1 << shft;
+  KMP_MB();
   if ((flag & pr_buf->th_doacross_flags[iter_number]) == 0)
     KMP_TEST_THEN_OR32(&pr_buf->th_doacross_flags[iter_number], flag);
   KA_TRACE(20, ("__kmpc_doacross_post() exit: T#%d iter %lld posted\n", gtid,
                 (iter_number << 5) + shft));
 }
 
 void __kmpc_doacross_fini(ident_t *loc, int gtid) {
   kmp_int64 num_done;

openmp/trunk/runtime/test/worksharing/for/kmp_doacross_check.c

int main()
   int i;
   int iter[N];
   struct dim dims;
   for( i = 0; i < N; ++i )
     iter[i] = 1;
   dims.lo = 1;
   dims.up = N-1;
   dims.st = 1;
-  #pragma omp parallel
+  #pragma omp parallel num_threads(4)
   {
     int i, gtid;
     long long vec;
     gtid = __kmpc_global_thread_num(NULL);
     __kmpc_doacross_init(NULL,gtid,1,&dims); // thread starts the loop
     #pragma omp for nowait schedule(dynamic)
     for( i = 1; i < N; ++i )
     {