[OpenMP][host runtime] Add initial hybrid CPU support

[jlpeyton]

Summary

Detect, through CPUID.1A, and show user different core types through KMP_AFFINITY=verbose mechanism. Offer future runtime optimizations __kmp_is_hybrid_cpu() to know whether running on a hybrid system or not.

[Nawrin]

LGTM

[jdoerfert]

openmp/runtime/src/kmp_affinity.h

601–607

Move this into the enum directly (as an enum value). Much easier to realize they belong together.

609–611

613

openmp/runtime/src/kmp_utility.cpp

249–250

This looks unrelated to "hybrid CPU" changes, no?

[jlpeyton]

Addressed patch comments

Moved small fixup of RTM CPUID bit check into small commit that prefixes this one:

diff --git a/openmp/runtime/src/kmp_utility.cpp b/openmp/runtime/src/kmp_utility.cpp
index bf3e6114e0c2..8118548c162c 100644
--- a/openmp/runtime/src/kmp_utility.cpp
+++ b/openmp/runtime/src/kmp_utility.cpp
@@ -247,15 +247,15 @@ void __kmp_query_cpuid(kmp_cpuinfo_t *p) {
                 i, buf.eax, buf.ebx, buf.ecx, buf.edx));
     }
 #endif
-#if KMP_USE_ADAPTIVE_LOCKS
     p->flags.rtm = 0;
     if (max_arg > 7) {
       /* RTM bit CPUID.07:EBX, bit 11 */
       __kmp_x86_cpuid(7, 0, &buf);
       p->flags.rtm = (buf.ebx >> 11) & 1;
-      KA_TRACE(trace_level, (" RTM"));
+      if (p->flags.rtm) {
+        KA_TRACE(trace_level, (" RTM"));
+      }
     }
-#endif
   }

   { // Parse CPU brand string for frequency, saving the string for later.

[jlpeyton]

@jdoerfert , does it look OK now?

[tlwilmar]

LGTM

openmp/runtime/src/i18n/en_US.txt

 TopologyExtraTile            "%1$s: %2$d packages x %3$d tiles/pkg x %4$d cores/tile x %5$d threads/core (%6$d total cores)"
 TopologyExtraNode            "%1$s: %2$d packages x %3$d nodes/pkg x %4$d cores/node x %5$d threads/core (%6$d total cores)"
 TopologyExtraNoTi            "%1$s: %2$d packages x %3$d nodes/pkg x %4$d tiles/node x %5$d cores/tile x %6$d threads/core (%7$d total cores)"
 OmptOutdatedWorkshare        "OMPT: Cannot determine workshare type; using the default (loop) instead. "
                              "This issue is fixed in an up-to-date compiler."
 OmpNoAllocator               "Allocator %1$s is not available, will use default allocator."
 TopologyGeneric              "%1$s: %2$s (%3$d total cores)"
 AffGranularityBad            "%1$s: granularity setting: %2$s does not exist in topology.  Using granularity=%3$s instead."
+TopologyHybrid               "%1$s: hybrid core type detected: %2$d %3$s cores."
 
 # --- OpenMP errors detected at runtime ---
 #
 #    %1 is the name of OpenMP construct (formatted with "Pragma" format).
 #
 CnsBoundToWorksharing        "%1$s must be bound to a work-sharing or work-queuing construct with an \"ordered\" clause"
 CnsDetectedEnd               "Detected end of %1$s without first executing a corresponding beginning."
 CnsIterationRangeTooLarge    "Iteration range too large in %1$s."

openmp/runtime/src/kmp.h

typedef struct kmp_cpuid {
   kmp_uint32 ebx;
   kmp_uint32 ecx;
   kmp_uint32 edx;
 } kmp_cpuid_t;
 
 typedef struct kmp_cpuinfo_flags_t {
   unsigned sse2 : 1; // 0 if SSE2 instructions are not supported, 1 otherwise.
   unsigned rtm : 1; // 0 if RTM instructions are not supported, 1 otherwise.
-  unsigned reserved : 30; // Ensure size of 32 bits
+  unsigned hybrid : 1;
+  unsigned reserved : 29; // Ensure size of 32 bits
 } kmp_cpuinfo_flags_t;
 
 typedef struct kmp_cpuinfo {
   int initialized; // If 0, other fields are not initialized.
   int signature; // CPUID(1).EAX
   int family; // CPUID(1).EAX[27:20]+CPUID(1).EAX[11:8] (Extended Family+Family)
   int model; // ( CPUID(1).EAX[19:16] << 4 ) + CPUID(1).EAX[7:4] ( ( Extended
   // Model << 4 ) + Model)

 extern int
     __kmp_storage_map; /* True means print storage map for threads and teams */
 extern int __kmp_storage_map_verbose; /* True means storage map includes
                                          placement info */
 extern int __kmp_storage_map_verbose_specified;
 
 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
 extern kmp_cpuinfo_t __kmp_cpuinfo;
+static inline bool __kmp_is_hybrid_cpu() { return __kmp_cpuinfo.flags.hybrid; }
+#else
+static inline bool __kmp_is_hybrid_cpu() { return false; }
 #endif
 
 extern volatile int __kmp_init_serial;
 extern volatile int __kmp_init_gtid;
 extern volatile int __kmp_init_common;
 extern volatile int __kmp_init_middle;
 extern volatile int __kmp_init_parallel;
 #if KMP_USE_MONITOR

openmp/runtime/src/kmp_affinity.h

KMPAffinity::Mask *index_mask_array(KMPAffinity::Mask *array,
     Mask *windows_array = static_cast<Mask *>(array);
     return &(windows_array[index]);
   }
   api_type get_api_type() const override { return NATIVE_OS; }
 };
 #endif /* KMP_OS_WINDOWS */
 #endif /* KMP_AFFINITY_SUPPORTED */
 
+typedef enum kmp_hw_core_type_t {
+  KMP_HW_CORE_TYPE_UNKNOWN = 0x0,
+#if KMP_ARCH_X86 || KMP_ARCH_X86_64
+  KMP_HW_CORE_TYPE_ATOM = 0x20,
+  KMP_HW_CORE_TYPE_CORE = 0x40,
+  KMP_HW_MAX_NUM_CORE_TYPES = 3,
+#else
+  KMP_HW_MAX_NUM_CORE_TYPES = 1,
+#endif
+} kmp_hw_core_type_t;
+
 class kmp_hw_thread_t {
 public:
   static const int UNKNOWN_ID = -1;
   static int compare_ids(const void *a, const void *b);
   static int compare_compact(const void *a, const void *b);
   int ids[KMP_HW_LAST];
   int sub_ids[KMP_HW_LAST];
   bool leader;
   int os_id;
+  kmp_hw_core_type_t core_type;
+
   void print() const;
   void clear() {
     for (int i = 0; i < (int)KMP_HW_LAST; ++i)
       ids[i] = UNKNOWN_ID;
     leader = false;
+    core_type = KMP_HW_CORE_TYPE_UNKNOWN;
   }
 };
 
 class kmp_topology_t {
 
   struct flags_t {
     int uniform : 1;
     int reserved : 31;
class kmp_topology_t {
   // Keep quick topology ratios, for non-uniform topologies,
   // this ratio holds the max number of itemAs per itemB
   // e.g., [ 4 packages | 6 cores / package | 2 threads / core ]
   int *ratio;
 
   // Storage containing the absolute number of each topology layer
   int *count;
 
+  // Storage containing the core types and the number of
+  // each core type for hybrid processors
+  kmp_hw_core_type_t core_types[KMP_HW_MAX_NUM_CORE_TYPES];
+  int core_types_count[KMP_HW_MAX_NUM_CORE_TYPES];
+
   // The hardware threads array
   // hw_threads is num_hw_threads long
   // Each hw_thread's ids and sub_ids are depth deep
   int num_hw_threads;
   kmp_hw_thread_t *hw_threads;
 
   // Equivalence hash where the key is the hardware topology item
   // and the value is the equivalent hardware topology type in the
class kmp_topology_t {
   // Set global affinity variables describing the number of threads per
   // core, the number of packages, the number of cores per package, and
   // the number of cores.
   void _set_globals();
 
   // Set the last level cache equivalent type
   void _set_last_level_cache();
 
+  // Increments the number of cores of type 'type'
+  void _increment_core_type(kmp_hw_core_type_t type) {
+    for (int i = 0; i < KMP_HW_MAX_NUM_CORE_TYPES; ++i) {
+      if (core_types[i] == KMP_HW_CORE_TYPE_UNKNOWN) {
+        core_types[i] = type;
+        core_types_count[i] = 1;
+        break;
+      } else if (core_types[i] == type) {
+        core_types_count[i]++;
+        break;
+      }
+    }
+  }
+
 public:
   // Force use of allocate()/deallocate()
   kmp_topology_t() = delete;
   kmp_topology_t(const kmp_topology_t &t) = delete;
   kmp_topology_t(kmp_topology_t &&t) = delete;
   kmp_topology_t &operator=(const kmp_topology_t &t) = delete;
   kmp_topology_t &operator=(kmp_topology_t &&t) = delete;
 

openmp/runtime/src/kmp_affinity.cpp

const char *__kmp_hw_get_keyword(kmp_hw_t type, bool plural) {
   case KMP_HW_THREAD:
     return ((plural) ? "threads" : "thread");
   case KMP_HW_PROC_GROUP:
     return ((plural) ? "proc_groups" : "proc_group");
   }
   return ((plural) ? "unknowns" : "unknown");
 }
 
+const char *__kmp_hw_get_core_type_string(kmp_hw_core_type_t type) {
+  switch (type) {
+  case KMP_HW_CORE_TYPE_UNKNOWN:
+    return "unknown";
+#if KMP_ARCH_X86 || KMP_ARCH_X86_64
+  case KMP_HW_CORE_TYPE_ATOM:
+    return "Intel Atom(R) processor";
+  case KMP_HW_CORE_TYPE_CORE:
+    return "Intel(R) Core(TM) processor";
+#endif
+  }
+  return "unknown";
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 // kmp_hw_thread_t methods
 int kmp_hw_thread_t::compare_ids(const void *a, const void *b) {
   const kmp_hw_thread_t *ahwthread = (const kmp_hw_thread_t *)a;
   const kmp_hw_thread_t *bhwthread = (const kmp_hw_thread_t *)b;
   int depth = __kmp_topology->get_depth();
   for (int level = 0; level < depth; ++level) {
     if (ahwthread->ids[level] < bhwthread->ids[level])

 #endif
 
 void kmp_hw_thread_t::print() const {
   int depth = __kmp_topology->get_depth();
   printf("%4d ", os_id);
   for (int i = 0; i < depth; ++i) {
     printf("%4d ", ids[i]);
   }
+  if (core_type != KMP_HW_CORE_TYPE_UNKNOWN) {
+    printf(" (%s)", __kmp_hw_get_core_type_string(core_type));
+  }
   printf("\n");
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 // kmp_topology_t methods
 
 // Remove layers that don't add information to the topology.
 // This is done by having the layer take on the id = UNKNOWN_ID (-1)
#endif
   }
   KMP_ASSERT(get_equivalent_type(KMP_HW_LLC) != KMP_HW_UNKNOWN);
 }
 
 // Gather the count of each topology layer and the ratio
 void kmp_topology_t::_gather_enumeration_information() {
   int previous_id[KMP_HW_LAST];
   int max[KMP_HW_LAST];
+  int previous_core_id = kmp_hw_thread_t::UNKNOWN_ID;
 
   for (int i = 0; i < depth; ++i) {
     previous_id[i] = kmp_hw_thread_t::UNKNOWN_ID;
     max[i] = 0;
     count[i] = 0;
     ratio[i] = 0;
   }
+  if (__kmp_is_hybrid_cpu()) {
+    for (int i = 0; i < KMP_HW_MAX_NUM_CORE_TYPES; ++i) {
+      core_types_count[i] = 0;
+      core_types[i] = KMP_HW_CORE_TYPE_UNKNOWN;
+    }
+  }
   for (int i = 0; i < num_hw_threads; ++i) {
     kmp_hw_thread_t &hw_thread = hw_threads[i];
     for (int layer = 0; layer < depth; ++layer) {
       int id = hw_thread.ids[layer];
       if (id != previous_id[layer]) {
         // Add an additional increment to each count
         for (int l = layer; l < depth; ++l)
           count[l]++;
         // Keep track of topology layer ratio statistics
         max[layer]++;
         for (int l = layer + 1; l < depth; ++l) {
           if (max[l] > ratio[l])
             ratio[l] = max[l];
           max[l] = 1;
         }
         break;
       }
     }
     for (int layer = 0; layer < depth; ++layer) {
       previous_id[layer] = hw_thread.ids[layer];
     }
+    // Figure out the number of each core type for hybrid CPUs
+    if (__kmp_is_hybrid_cpu()) {
+      int core_level = get_level(KMP_HW_CORE);
+      if (core_level != -1) {
+        if (hw_thread.ids[core_level] != previous_core_id)
+          _increment_core_type(hw_thread.core_type);
+        previous_core_id = hw_thread.ids[core_level];
+      }
+    }
   }
   for (int layer = 0; layer < depth; ++layer) {
     if (max[layer] > ratio[layer])
       ratio[layer] = max[layer];
   }
 }
 
 // Find out if the topology is uniform
void kmp_topology_t::dump() const {
   printf("\n");
 
   printf("* count: ");
   for (int i = 0; i < depth; ++i) {
     printf("%15d ", count[i]);
   }
   printf("\n");
 
+  printf("* core_types:\n");
+  for (int i = 0; i < KMP_HW_MAX_NUM_CORE_TYPES; ++i) {
+    if (core_types[i] != KMP_HW_CORE_TYPE_UNKNOWN) {
+      printf("    %d %s core%c\n", core_types_count[i],
+             __kmp_hw_get_core_type_string(core_types[i]),
+             ((core_types_count[i] > 1) ? 's' : ' '));
+    } else {
+      if (i == 0)
+        printf("No hybrid information available\n");
+      break;
+    }
+  }
+
   printf("* equivalent map:\n");
   KMP_FOREACH_HW_TYPE(i) {
     const char *key = __kmp_hw_get_keyword(i);
     const char *value = __kmp_hw_get_keyword(equivalent[i]);
     printf("%-15s -> %-15s\n", key, value);
   }
 
   printf("* uniform: %s\n", (is_uniform() ? "Yes" : "No"));
if (plevel == 0) {
       __kmp_str_buf_print(&buf, " x %d %s/%s", c,
                           __kmp_hw_get_catalog_string(numerator_type, plural),
                           __kmp_hw_get_catalog_string(denominator_type));
     }
     denominator_type = numerator_type;
   }
   KMP_INFORM(TopologyGeneric, env_var, buf.str, ncores);
 
+  if (__kmp_is_hybrid_cpu()) {
+    for (int i = 0; i < KMP_HW_MAX_NUM_CORE_TYPES; ++i) {
+      if (core_types[i] == KMP_HW_CORE_TYPE_UNKNOWN)
+        break;
+      KMP_INFORM(TopologyHybrid, env_var, core_types_count[i],
+                 __kmp_hw_get_core_type_string(core_types[i]));
+    }
+  }
+
   if (num_hw_threads <= 0) {
     __kmp_str_buf_free(&buf);
     return;
   }
 
   // Full OS proc to hardware thread map
   KMP_INFORM(OSProcToPhysicalThreadMap, env_var);
   for (int i = 0; i < num_hw_threads; i++) {
     __kmp_str_buf_clear(&buf);
     for (int level = 0; level < depth; ++level) {
       kmp_hw_t type = types[level];
       __kmp_str_buf_print(&buf, "%s ", __kmp_hw_get_catalog_string(type));
       __kmp_str_buf_print(&buf, "%d ", hw_threads[i].ids[level]);
     }
+    if (__kmp_is_hybrid_cpu())
+      __kmp_str_buf_print(
+          &buf, "(%s)", __kmp_hw_get_core_type_string(hw_threads[i].core_type));
     KMP_INFORM(OSProcMapToPack, env_var, hw_threads[i].os_id, buf.str);
   }
 
   __kmp_str_buf_free(&buf);
 }
 
 void kmp_topology_t::canonicalize() {
   _remove_radix1_layers();
if (!__kmp_topology->check_ids()) {
     kmp_topology_t::deallocate(__kmp_topology);
     __kmp_topology = nullptr;
     *msg_id = kmp_i18n_str_LegacyApicIDsNotUnique;
     return false;
   }
   return true;
 }
 
+// Hybrid cpu detection using CPUID.1A
+// Thread should be pinned to processor already
+static void __kmp_get_hybrid_info(kmp_hw_core_type_t *type,
+                                  unsigned *native_model_id) {
+  kmp_cpuid buf;
+  __kmp_x86_cpuid(0x1a, 0, &buf);
+  *type = (kmp_hw_core_type_t)__kmp_extract_bits<24, 31>(buf.eax);
+  *native_model_id = __kmp_extract_bits<0, 23>(buf.eax);
+}
+
 // Intel(R) microarchitecture code name Nehalem, Dunnington and later
 // architectures support a newer interface for specifying the x2APIC Ids,
 // based on CPUID.B or CPUID.1F
 /*
  * CPUID.B or 1F, Input ECX (sub leaf # aka level number)
     Bits            Bits            Bits           Bits
     31-16           15-8            7-4            4-0
 ---+-----------+--------------+-------------+-----------------+
KMP_CPU_SET_ITERATE(proc, __kmp_affin_fullMask) {
     hw_thread.os_id = proc;
     // Put in topology information
     for (unsigned j = 0, idx = depth - 1; j < my_levels_index; ++j, --idx) {
       hw_thread.ids[idx] = apic_id & my_levels[j].mask;
       if (j > 0) {
         hw_thread.ids[idx] >>= my_levels[j - 1].mask_width;
       }
     }
+    // Hybrid information
+    if (__kmp_is_hybrid_cpu() && highest_leaf >= 0x1a) {
+      kmp_hw_core_type_t type;
+      unsigned native_model_id;
+      __kmp_get_hybrid_info(&type, &native_model_id);
+      hw_thread.core_type = type;
+    }
     hw_thread_index++;
   }
   KMP_ASSERT(hw_thread_index > 0);
   __kmp_topology->sort_ids();
   if (!__kmp_topology->check_ids()) {
     kmp_topology_t::deallocate(__kmp_topology);
     __kmp_topology = nullptr;
     *msg_id = kmp_i18n_str_x2ApicIDsNotUnique;

openmp/runtime/src/kmp_utility.cpp

#ifdef KMP_DEBUG
     KA_TRACE(trace_level, (" ]\n"));
 
     for (i = 2; i <= max_arg; ++i) {
       __kmp_x86_cpuid(i, 0, &buf);
       KA_TRACE(trace_level,
                ("INFO: CPUID %d: EAX=0x%08X EBX=0x%08X ECX=0x%08X EDX=0x%08X\n",
                 i, buf.eax, buf.ebx, buf.ecx, buf.edx));
     }
 #endif
     p->flags.rtm = 0;
+    p->flags.hybrid = 0;
     if (max_arg > 7) {
       /* RTM bit CPUID.07:EBX, bit 11 */
+      /* HYRBID bit CPUID.07:EDX, bit 15 */
       __kmp_x86_cpuid(7, 0, &buf);
       p->flags.rtm = (buf.ebx >> 11) & 1;
+      p->flags.hybrid = (buf.edx >> 15) & 1;
       if (p->flags.rtm) {
         KA_TRACE(trace_level, (" RTM"));
       }
+      if (p->flags.hybrid) {
+        KA_TRACE(trace_level, (" HYBRID"));
+      }
     }
   }
 
   { // Parse CPU brand string for frequency, saving the string for later.
     int i;
     kmp_cpuid_t *base = (kmp_cpuid_t *)&p->name[0];
 
     // Get CPU brand string.