Fix affinity mask usage on big-endian CPUs

[tstellar]

Summary

This fixes almost all the lit tests on ppc64 big-endian. Prior to this patch,
the affinity masks were printed like this on a 8-core system:

OMP: Info #154: KMP_AFFINITY: Initial OS proc set respected: {56,57,58,59,60,61,62,63}

[jlpeyton]

To avoid the glibc version issue, can we just switch the mask_t from unsigned char to unsigned long? This is the actual underlying type for cpuset according to the man page. This would require different changes, particularly __kmp_affin_mask_size. I think that in z_Linux_util.cpp:__kmp_determine_affinity_capable() (where __kmp_affin_mask_size is set), you could change __kmp_affin_mask_size to be the closest multiple of sizeof(unsigned long), then the Mask() constructor would perform __kmp_allocate(sizeof(unsigned long)*__kmp_affin_mask_size);. Most of the mask API stays the same, but in set|get_system_affinity(), have the size argument be sizeof(unsigned long)*__kmp_affin_mask_size for the system calls.

Does this make sense?

[tstellar]

Is this more what you had in mind?

[jlpeyton]

Almost, if KMP_AFFINITY_ENABLE() is reverted back and if the KMPNativeAffinity::Mask::mask_t type was made public (I don't see any problem doing that), then along with the rest of changes you made, I think it should look something like this:

diff --git a/runtime/src/z_Linux_util.cpp b/runtime/src/z_Linux_util.cpp
index 8c59b43..7506380 100644
--- a/runtime/src/z_Linux_util.cpp
+++ b/runtime/src/z_Linux_util.cpp
@@ -179,7 +179,10 @@ void __kmp_affinity_determine_capable(const char *env_var) {
         KMP_INTERNAL_FREE(buf);
       }
       if (errno == EFAULT) {
-        KMP_AFFINITY_ENABLE(gCode);
+        int mask_size = ((gCode + sizeof(KMPNativeAffinity::Mask::mask_t) - 1) &
+                         ~(sizeof(KMPNativeAffinity::Mask::mask_t))) /
+                        sizeof(KMPNativeAffinity::Mask::mask_t);
+        KMP_AFFINITY_ENABLE(mask_size);
         KA_TRACE(10, ("__kmp_affinity_determine_capable: "
                       "affinity supported (mask size %d)\n",
                       (int)__kmp_affin_mask_size));
@@ -256,7 +259,10 @@ void __kmp_affinity_determine_capable(const char *env_var) {
         return;
       }
       if (errno == EFAULT) {
-        KMP_AFFINITY_ENABLE(gCode);
+        int mask_size = ((gCode + sizeof(KMPNativeAffinity::Mask::mask_t) - 1) &
+                         ~(sizeof(KMPNativeAffinity::Mask::mask_t))) /
+                        sizeof(KMPNativeAffinity::Mask::mask_t);
+        KMP_AFFINITY_ENABLE(mask_size);
         KA_TRACE(10, ("__kmp_affinity_determine_capable: "
                       "affinity supported (mask size %d)\n",
                       (int)__kmp_affin_mask_size));

Sorry, I didn't explain it all the way in my previous comment. __kmp_affin_mask_size should be the number of mask_ts to allocate (instead of just the raw size in bytes) which requires further dividing by sizeof(mask_t). So if we take the code you have in the macro and apply it to the two places in z_Linux_util.cpp (plus the extra division), then it would look like above. This also allows us to change mask_t to whatever type we please without any further modifications to the code. Also, this won't disturb the Windows/Hwloc code if we leave the KMP_AFFINITY_ENABLE() macro alone.

So

1. Revert KMP_AFFINITY_ENABLE()
2. Keep the rest of your current changes
3. Make mask_t public
4. Add the changes to z_Linux_util.cpp

[jlpeyton]

Changing state to request changes.

runtime/src/kmp.h

 extern int __kmp_hwloc_error;
 extern int __kmp_numa_detected;
 extern int __kmp_tile_depth;
 #endif
 
 extern size_t __kmp_affin_mask_size;
 #define KMP_AFFINITY_CAPABLE() (__kmp_affin_mask_size > 0)
 #define KMP_AFFINITY_DISABLE() (__kmp_affin_mask_size = 0)
-#define KMP_AFFINITY_ENABLE(mask_size) (__kmp_affin_mask_size = mask_size)
+// Round to the nearest multiple of sizeof(unsigned long), which is the
+// size of mask_t.
+#define KMP_AFFINITY_ENABLE(mask_size) \
+  (__kmp_affin_mask_size = ((mask_size + sizeof(unsigned long) - 1) &          \
+                           ~(sizeof(unsigned long)-1)))
 #define KMP_CPU_SET_ITERATE(i, mask)                                           \
   for (i = (mask)->begin(); (int)i != (mask)->end(); i = (mask)->next(i))
 #define KMP_CPU_SET(i, mask) (mask)->set(i)
 #define KMP_CPU_ISSET(i, mask) (mask)->is_set(i)
 #define KMP_CPU_CLR(i, mask) (mask)->clear(i)
 #define KMP_CPU_ZERO(mask) (mask)->zero()
 #define KMP_CPU_COPY(dest, src) (dest)->copy(src)
 #define KMP_CPU_AND(dest, src) (dest)->bitwise_and(src)

runtime/src/kmp_affinity.h

 #define __NR_sched_getaffinity 5196
 #elif __NR_sched_getaffinity != 5196
 #error Wrong code for getaffinity system call.
 #endif /* __NR_sched_getaffinity */
 #error Unknown or unsupported architecture
 #endif /* KMP_ARCH_* */
 class KMPNativeAffinity : public KMPAffinity {
   class Mask : public KMPAffinity::Mask {
-    typedef unsigned char mask_t;
+    typedef unsigned long mask_t;
     static const int BITS_PER_MASK_T = sizeof(mask_t) * CHAR_BIT;
 
   public:
     mask_t *mask;
-    Mask() { mask = (mask_t *)__kmp_allocate(__kmp_affin_mask_size); }
+    Mask() { mask = (mask_t *)__kmp_allocate(sizeof(mask_t) * __kmp_affin_mask_size); }
     ~Mask() {
       if (mask)
         __kmp_free(mask);
     }
     void set(int i) override {
       mask[i / BITS_PER_MASK_T] |= ((mask_t)1 << (i % BITS_PER_MASK_T));
     }
     bool is_set(int i) const override {
int next(int previous) const override {
       while (retval < end() && !is_set(retval))
         ++retval;
       return retval;
     }
     int get_system_affinity(bool abort_on_error) override {
       KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
                   "Illegal get affinity operation when not capable");
       int retval =
-          syscall(__NR_sched_getaffinity, 0, __kmp_affin_mask_size, mask);
+          syscall(__NR_sched_getaffinity, 0, sizeof(mask_t) * __kmp_affin_mask_size, mask);
       if (retval >= 0) {
         return 0;
       }
       int error = errno;
       if (abort_on_error) {
         __kmp_fatal(KMP_MSG(FatalSysError), KMP_ERR(error), __kmp_msg_null);
       }
       return error;
     }
     int set_system_affinity(bool abort_on_error) const override {
       KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
                   "Illegal get affinity operation when not capable");
       int retval =
-          syscall(__NR_sched_setaffinity, 0, __kmp_affin_mask_size, mask);
+          syscall(__NR_sched_setaffinity, 0, sizeof(mask_t) * __kmp_affin_mask_size, mask);
       if (retval >= 0) {
         return 0;
       }
       int error = errno;
       if (abort_on_error) {
         __kmp_fatal(KMP_MSG(FatalSysError), KMP_ERR(error), __kmp_msg_null);
       }
       return error;