Fix for CMPLRS-48319 proc_bind affinity bug

[jlpeyton]

Summary

Using proc_bind clause on a nested #pragma omp parallel region
with KMP_AFFINITY set causes an assertion error. This assertion occurs because
the place-partition-var is not properly initialized in the nested master threads.
Trying to get an intuitive result with KMP_AFFINITY + proc_bind is difficult
because of how the KMP_AFFINITY gtid-to-place mapping occurs. This
patch creates an initial place list no matter what affinity mechanism is used.
For KMP_AFFINITY, the place-partition-var is initialized to all the places.

[tlwilmar]

LGTM

openmp/trunk/runtime/src/kmp_affinity.cpp

#if OMP_40_ENABLED
     }
 #endif
     if (__kmp_affinity_num_masks == 0) {
       if (__kmp_affinity_verbose ||
           (__kmp_affinity_warnings && (__kmp_affinity_type != affinity_none))) {
         KMP_WARNING(AffNoValidProcID);
       }
       __kmp_affinity_type = affinity_none;
+      __kmp_create_affinity_none_places();
       return;
     }
     break;
 
   // The other affinity types rely on sorting the Addresses according to some
   // permutation of the machine topology tree. Set __kmp_affinity_compact and
   // __kmp_affinity_offset appropriately, then jump to a common code fragment
   // to do the sort and create the array of affinity masks.
case affinity_compact:
     goto sortAddresses;
 
   case affinity_balanced:
     if (depth <= 1) {
       if (__kmp_affinity_verbose || __kmp_affinity_warnings) {
         KMP_WARNING(AffBalancedNotAvail, "KMP_AFFINITY");
       }
       __kmp_affinity_type = affinity_none;
+      __kmp_create_affinity_none_places();
       return;
-    } else if (__kmp_affinity_uniform_topology()) {
-      break;
-    } else { // Non-uniform topology
-
+    } else if (!__kmp_affinity_uniform_topology()) {
       // Save the depth for further usage
       __kmp_aff_depth = depth;
 
       int core_level = __kmp_affinity_find_core_level(
           address2os, __kmp_avail_proc, depth - 1);
       int ncores = __kmp_affinity_compute_ncores(address2os, __kmp_avail_proc,
                                                  depth - 1, core_level);
       int maxprocpercore = __kmp_affinity_max_proc_per_core(
if (depth <= 1) {
           inlastcore++;
         } else {
           inlastcore = 0;
         }
         lastcore = core;
 
         procarr[core * maxprocpercore + inlastcore] = proc;
       }
-
-      break;
+    }
+    if (__kmp_affinity_compact >= depth) {
+      __kmp_affinity_compact = depth - 1;
     }
 
   sortAddresses:
     // Allocate the gtid->affinity mask table.
     if (__kmp_affinity_dups) {
       __kmp_affinity_num_masks = __kmp_avail_proc;
     } else {
       __kmp_affinity_num_masks = numUnique;

 #endif
 
 #if OMP_40_ENABLED
   th->th.th_current_place = i;
   if (isa_root) {
     th->th.th_new_place = i;
     th->th.th_first_place = 0;
     th->th.th_last_place = __kmp_affinity_num_masks - 1;
+  } else if (KMP_AFFINITY_NON_PROC_BIND) {
+    // When using a Non-OMP_PROC_BIND affinity method,
+    // set all threads' place-partition-var to the entire place list
+    th->th.th_first_place = 0;
+    th->th.th_last_place = __kmp_affinity_num_masks - 1;
   }
 
   if (i == KMP_PLACE_ALL) {
     KA_TRACE(100, ("__kmp_affinity_set_init_mask: binding T#%d to all places\n",
                    gtid));
   } else {
     KA_TRACE(100, ("__kmp_affinity_set_init_mask: binding T#%d to place %d\n",
                    gtid, i));

openmp/trunk/runtime/src/kmp_ftn_entry.h

 #else
   int gtid, num_places, first_place, last_place;
   kmp_info_t *thread;
   if (!TCR_4(__kmp_init_middle)) {
     __kmp_middle_initialize();
   }
   if (!KMP_AFFINITY_CAPABLE())
     return 0;
-  if (KMP_AFFINITY_NON_PROC_BIND) {
-    return 1;
-  }
   gtid = __kmp_entry_gtid();
   thread = __kmp_thread_from_gtid(gtid);
   first_place = thread->th.th_first_place;
   last_place = thread->th.th_last_place;
   if (first_place < 0 || last_place < 0)
     return 0;
   if (first_place <= last_place)
     num_places = last_place - first_place + 1;
#else
   kmp_info_t *thread;
   if (!TCR_4(__kmp_init_middle)) {
     __kmp_middle_initialize();
   }
   if (!KMP_AFFINITY_CAPABLE())
     return;
   gtid = __kmp_entry_gtid();
   thread = __kmp_thread_from_gtid(gtid);
-  if (KMP_AFFINITY_NON_PROC_BIND) {
-    place_nums[0] = thread->th.th_current_place;
-    return;
-  }
   first_place = thread->th.th_first_place;
   last_place = thread->th.th_last_place;
   if (first_place < 0 || last_place < 0)
     return;
   if (first_place <= last_place) {
     start = first_place;
     end = last_place;
   } else {

openmp/trunk/runtime/test/affinity/bug-nested.c

+// RUN: %libomp-compile && env KMP_AFFINITY=compact %libomp-run
+// REQUIRES: openmp-4.0
+
+#include <stdio.h>
+#include <stdint.h>
+#include <omp.h>
+#include "omp_testsuite.h"
+
+int test_nested_affinity_bug() {
+  int a = 0;
+  omp_set_nested(1);
+  #pragma omp parallel num_threads(2) shared(a)
+  {
+    #pragma omp parallel num_threads(2) shared(a) proc_bind(close)
+    {
+      #pragma omp atomic
+      a++;
+    }
+  }
+  return 1;
+}
+
+int main() {
+  int i;
+  int num_failed = 0;
+
+  for (i = 0; i < REPETITIONS; i++) {
+    if (!test_nested_affinity_bug()) {
+      num_failed++;
+    }
+  }
+  return num_failed;
+}