[LIBOMPTARGET] Adding AMD to llvm-omp-device-info

[josemonsalve2]

Summary

Adding device information print for AMD devices on the
llvm-omp-device-info command line tool. The output is inspired by
the rocminfo command line tool.

This commit adds missing HSA functions, enums and structs
needed to query additional information from the HSA agents.
A generic message for the generic-elf-64bit plugin is also added

Example of an output:

llvm-omp-device-info
Device (0):
    This is a generic-elf-64bit device

Device (1):
    This is a generic-elf-64bit device

Device (2):
    This is a generic-elf-64bit device

Device (3):
    This is a generic-elf-64bit device

Device (4):
    HSA Runtime Version:                1.1
    HSA OpenMP Device Number:           0
    Device Name:                        gfx906
    Vendor Name:                        AMD
    Device Type:                        GPU
    Max Queues:                         128
    Queue Min Size:                     64
    Queue Max Size:                     131072
    Cache:
      L0:                               16384 bytes
      L1:                               8388608 bytes
    Cacheline Size:                     64
    Max Clock Freq(MHz):                1725
    Compute Units:                      60
    SIMD per CU:                        4
    Fast F16 Operation:                 TRUE
    Wavefront Size:                     64
    Workgroup Max Size:                 1024
    Workgroup Max Size per Dimension:
      x:                                1024
      y:                                1024
      z:                                1024
    Max Waves Per CU:                   40
    Max Work-item Per CU:               2560
    Grid Max Size:                      4294967295
    Grid Max Size per Dimension:
      x:                                4294967295
      y:                                4294967295
      z:                                4294967295
    Max fbarriers/Workgrp:              32
    Memory Pools:
      Pool GLOBAL; FLAGS: COARSE GRAINED, :
        Size:                            34342961152 bytes
        Allocatable:                     TRUE
        Runtime Alloc Granule:           4096 bytes
        Runtime Alloc alignment:         4096 bytes
        Accessable by all:               FALSE
      Pool GLOBAL; FLAGS: FINE GRAINED, :
        Size:                            34342961152 bytes
        Allocatable:                     TRUE
        Runtime Alloc Granule:           4096 bytes
        Runtime Alloc alignment:         4096 bytes
        Accessable by all:               FALSE
      Pool GROUP:
        Size:                            65536 bytes
        Allocatable:                     FALSE
        Runtime Alloc Granule:           0 bytes
        Runtime Alloc alignment:         0 bytes
        Accessable by all:               FALSE

Device (5):
    HSA Runtime Version:                1.1
    HSA OpenMP Device Number:           1
    Device Name:                        gfx906
    Vendor Name:                        AMD
    Device Type:                        GPU
    Max Queues:                         128
    Queue Min Size:                     64
    Queue Max Size:                     131072
    Cache:
      L0:                               16384 bytes
      L1:                               8388608 bytes
    Cacheline Size:                     64
    Max Clock Freq(MHz):                1725
    Compute Units:                      60
    SIMD per CU:                        4
    Fast F16 Operation:                 TRUE
    Wavefront Size:                     64
    Workgroup Max Size:                 1024
    Workgroup Max Size per Dimension:
      x:                                1024
      y:                                1024
      z:                                1024
    Max Waves Per CU:                   40
    Max Work-item Per CU:               2560
    Grid Max Size:                      4294967295
    Grid Max Size per Dimension:
      x:                                4294967295
      y:                                4294967295
      z:                                4294967295
    Max fbarriers/Workgrp:              32
    Memory Pools:
      Pool GLOBAL; FLAGS: COARSE GRAINED, :
        Size:                            34342961152 bytes
        Allocatable:                     TRUE
        Runtime Alloc Granule:           4096 bytes
        Runtime Alloc alignment:         4096 bytes
        Accessable by all:               FALSE
      Pool GLOBAL; FLAGS: FINE GRAINED, :
        Size:                            34342961152 bytes
        Allocatable:                     TRUE
        Runtime Alloc Granule:           4096 bytes
        Runtime Alloc alignment:         4096 bytes
        Accessable by all:               FALSE
      Pool GROUP:
        Size:                            65536 bytes
        Allocatable:                     FALSE
        Runtime Alloc Granule:           0 bytes
        Runtime Alloc alignment:         0 bytes
        Accessable by all:               FALSE

Device (6):
    HSA Runtime Version:                1.1
    HSA OpenMP Device Number:           2
    Device Name:                        gfx906
    Vendor Name:                        AMD
    Device Type:                        GPU
    Max Queues:                         128
    Queue Min Size:                     64
    Queue Max Size:                     131072
    Cache:
      L0:                               16384 bytes
      L1:                               8388608 bytes
    Cacheline Size:                     64
    Max Clock Freq(MHz):                1725
    Compute Units:                      60
    SIMD per CU:                        4
    Fast F16 Operation:                 TRUE
    Wavefront Size:                     64
    Workgroup Max Size:                 1024
    Workgroup Max Size per Dimension:
      x:                                1024
      y:                                1024
      z:                                1024
    Max Waves Per CU:                   40
    Max Work-item Per CU:               2560
    Grid Max Size:                      4294967295
    Grid Max Size per Dimension:
      x:                                4294967295
      y:                                4294967295
      z:                                4294967295
    Max fbarriers/Workgrp:              32
    Memory Pools:
      Pool GLOBAL; FLAGS: COARSE GRAINED, :
        Size:                            34342961152 bytes
        Allocatable:                     TRUE
        Runtime Alloc Granule:           4096 bytes
        Runtime Alloc alignment:         4096 bytes
        Accessable by all:               FALSE
      Pool GLOBAL; FLAGS: FINE GRAINED, :
        Size:                            34342961152 bytes
        Allocatable:                     TRUE
        Runtime Alloc Granule:           4096 bytes
        Runtime Alloc alignment:         4096 bytes
        Accessable by all:               FALSE
      Pool GROUP:
        Size:                            65536 bytes
        Allocatable:                     FALSE
        Runtime Alloc Granule:           0 bytes
        Runtime Alloc alignment:         0 bytes
        Accessable by all:               FALSE

Device (7):
    HSA Runtime Version:                1.1
    HSA OpenMP Device Number:           3
    Device Name:                        gfx906
    Vendor Name:                        AMD
    Device Type:                        GPU
    Max Queues:                         128
    Queue Min Size:                     64
    Queue Max Size:                     131072
    Cache:
      L0:                               16384 bytes
      L1:                               8388608 bytes
    Cacheline Size:                     64
    Max Clock Freq(MHz):                1725
    Compute Units:                      60
    SIMD per CU:                        4
    Fast F16 Operation:                 TRUE
    Wavefront Size:                     64
    Workgroup Max Size:                 1024
    Workgroup Max Size per Dimension:
      x:                                1024
      y:                                1024
      z:                                1024
    Max Waves Per CU:                   40
    Max Work-item Per CU:               2560
    Grid Max Size:                      4294967295
    Grid Max Size per Dimension:
      x:                                4294967295
      y:                                4294967295
      z:                                4294967295
    Max fbarriers/Workgrp:              32
    Memory Pools:
      Pool GLOBAL; FLAGS: COARSE GRAINED, :
        Size:                            34342961152 bytes
        Allocatable:                     TRUE
        Runtime Alloc Granule:           4096 bytes
        Runtime Alloc alignment:         4096 bytes
        Accessable by all:               FALSE
      Pool GLOBAL; FLAGS: FINE GRAINED, :
        Size:                            34342961152 bytes
        Allocatable:                     TRUE
        Runtime Alloc Granule:           4096 bytes
        Runtime Alloc alignment:         4096 bytes
        Accessable by all:               FALSE
      Pool GROUP:
        Size:                            65536 bytes
        Allocatable:                     FALSE
        Runtime Alloc Granule:           0 bytes
        Runtime Alloc alignment:         0 bytes
        Accessable by all:               FALSE

[JonChesterfield]

I think this is reasonable, assuming that's a feature we want from this tool. Not keen on the interleaving of HSA calls and printf as it makes it relatively difficult to read but the overall complexity is still low. Will leave it open for other comments but intend to accept.

edit: the formatted output is not easily parsable which is annoying but does look very like rocm-info, so maybe that's fine. I'd have been tempted to render it as JSON or similar by default, and add a helper that turns that format into the rocm-info format if requested. Would be more work than the current patch.

openmp/libomptarget/plugins/amdgpu/dynamic_hsa/hsa_ext_amd.h

62

Several of these fields are unused, not sure it's good to add them

openmp/libomptarget/plugins/amdgpu/src/rtl.cpp

555

Can this be static or a free function? Lots of state in it but I'm hopeful it doesn't mutate anything

[saiislam]

Could you please add support for HSA_ISA_INFO_NAME using hsa_isa_get_info_alt?
This one gives triple and target ID information, like amdgcn-amd-amdhsa--gfx908:sramecc-:xnack-

[josemonsalve2]

@saiislam I will look into that.

@JonChesterfield I look at the rocm-info code and they have different structs for all the different fields. I just didn't want to do all that, since it is not really that important for the runtime itself. However, the idea of the JSON file is good. Let's consider it and re-structure this.

One option could be to add a device info layer, with all the details, that can be access by the runtime, as well as output in text and JSON. But this should be an extension of this.

openmp/libomptarget/plugins/amdgpu/dynamic_hsa/hsa_ext_amd.h

62

Ok, I will remove the ones that are not used

openmp/libomptarget/plugins/amdgpu/src/rtl.cpp

555

That's reasonable. It should not mutate anything

[josemonsalve2]

I have added the HSA_ISA_INFO_NAME, made the function static, and removed
the unused elements in the enums.

[JonChesterfield]

openmp/libomptarget/plugins/amdgpu/dynamic_hsa/hsa_ext_amd.h

62

I guess that was unclear, I meant for all the enums. E.g. a brief manual search suggests HSA_REGION_INFO_RUNTIME_ALLOC_ALLOWED is dead. Would suggest commenting out all of the new ones then recompiling as a crude but quick way to produce the minimal set.

I could probably be persuaded we can have dead code here that happens to match the downstream hsa.h as it's supposed to be a stable interface but I have vague fears of the numbers diverging between the two and confusion resulting. Perhaps the right thing to do is create a test case that checks this file against a hsa.h and errors if they diverge.

[josemonsalve2]

I missed those. I will check well with all of them.

There's one problem with the hsa_cache_info_t that the version I am using is deprecated, but the one version that is current (commented in the code) breaks. It segfaults. I think it is best to use the deprecated version as this is used also by default in rocminfo. However, I am not sure if I should leave the commented code, and also the hsa_cache_info_t struct.

[JonChesterfield]

Thanks. The comment about using the deprecated interface because the new one segfaults is good, but the commented out code should go. It's straightforward to reimplement later if we wish to.

[josemonsalve2]

Cleaning up code to remove unused enums. Removed the commented code with cache_iteration, and all related funcitons.

[saiislam]

In D126836#3554498, @josemonsalve2 wrote:

I have added the HSA_ISA_INFO_NAME, made the function static, and removed
the unused elements in the enums.

Thank you!

[JonChesterfield]

I think that's all the comments addressed. Thanks!

[josemonsalve2]

Thanks Jon for all the comments.

There's an error that I think it is from clang-format, do I need to address it before merging?

I check and clang-format will change the order of two header files. It was not introduced by me.

Thanks

[jhuber6]

In D126836#3566283, @josemonsalve2 wrote:

Thanks Jon for all the comments.

There's an error that I think it is from clang-format, do I need to address it before merging?

I check and clang-format will change the order of two header files. It was not introduced by me.

Thanks

This should format only the portions you introduced or changed.

git clang-format HEAD~1

[arsenm]

Seems unfortunate that the vendor name reported by HSA isn't identical to the one reported by OpenCL

[josemonsalve2]

Running clang-format

[HaohaiWen]

openmp/libomptarget/plugins/amdgpu/src/rtl.cpp

750

openmp/libomptarget/plugins/amdgpu/src/rtl.cpp:750:7: error: default label in switch which covers all enumeration values [-Werror,-Wcovered-switch-default]
There's warning for this switch body.

[josemonsalve2]

Ah! Sorry about this. Quick fix. Working on it.

[josemonsalve2]

Fixing the warning issue on the switch statement.

openmp/libomptarget/plugins/amdgpu/dynamic_hsa/hsa.h

 
 typedef enum {
   HSA_DEVICE_TYPE_CPU = 0,
   HSA_DEVICE_TYPE_GPU = 1,
   HSA_DEVICE_TYPE_DSP = 2
 } hsa_device_type_t;
 
 typedef enum {
+  HSA_ISA_INFO_NAME = 1,
+} hsa_isa_info_t;
+
+typedef enum {
   HSA_AGENT_INFO_NAME = 0,
+  HSA_AGENT_INFO_VENDOR_NAME = 1,
   HSA_AGENT_INFO_PROFILE = 4,
   HSA_AGENT_INFO_WAVEFRONT_SIZE = 6,
   HSA_AGENT_INFO_WORKGROUP_MAX_DIM = 7,
+  HSA_AGENT_INFO_WORKGROUP_MAX_SIZE = 8,
   HSA_AGENT_INFO_GRID_MAX_DIM = 9,
+  HSA_AGENT_INFO_GRID_MAX_SIZE = 10,
+  HSA_AGENT_INFO_FBARRIER_MAX_SIZE = 11,
+  HSA_AGENT_INFO_QUEUES_MAX = 12,
+  HSA_AGENT_INFO_QUEUE_MIN_SIZE = 13,
   HSA_AGENT_INFO_QUEUE_MAX_SIZE = 14,
   HSA_AGENT_INFO_DEVICE = 17,
+  HSA_AGENT_INFO_CACHE_SIZE = 18,
+  HSA_AGENT_INFO_FAST_F16_OPERATION = 24,
 } hsa_agent_info_t;
 
+typedef enum {
+  HSA_SYSTEM_INFO_VERSION_MAJOR = 0,
+  HSA_SYSTEM_INFO_VERSION_MINOR = 1,
+} hsa_system_info_t;
+
+typedef struct hsa_region_s {
+  uint64_t handle;
+} hsa_region_t;
+
+typedef struct hsa_cache_s {
+  uint64_t handle;
+} hsa_cache_t;
+
+typedef struct hsa_isa_s {
+  uint64_t handle;
+} hsa_isa_t;
+
+typedef enum {
+  HSA_REGION_INFO_SEGMENT = 0,
+  HSA_REGION_INFO_GLOBAL_FLAGS = 1,
+  HSA_REGION_INFO_SIZE = 2,
+  HSA_REGION_INFO_ALLOC_MAX_SIZE = 4,
+  HSA_REGION_INFO_ALLOC_MAX_PRIVATE_WORKGROUP_SIZE = 8,
+  HSA_REGION_INFO_RUNTIME_ALLOC_ALLOWED = 5,
+  HSA_REGION_INFO_RUNTIME_ALLOC_GRANULE = 6,
+  HSA_REGION_INFO_RUNTIME_ALLOC_ALIGNMENT = 7
+} hsa_region_info_t;
+
+typedef enum {
+  HSA_CACHE_INFO_NAME_LENGTH = 0,
+  HSA_CACHE_INFO_NAME = 1,
+  HSA_CACHE_INFO_LEVEL = 2,
+  HSA_CACHE_INFO_SIZE = 3
+} hsa_cache_info_t;
+
+hsa_status_t hsa_system_get_info(hsa_system_info_t attribute, void *value);
+
 hsa_status_t hsa_agent_get_info(hsa_agent_t agent, hsa_agent_info_t attribute,
                                 void *value);
 
+hsa_status_t hsa_cache_get_info(hsa_cache_t cache, hsa_cache_info_t attribute,
+                                void *value);
+
+hsa_status_t  hsa_isa_get_info_alt(hsa_isa_t isa, hsa_isa_info_t attribute,
+                                void *value);
+
 hsa_status_t hsa_iterate_agents(hsa_status_t (*callback)(hsa_agent_t agent,
                                                          void *data),
                                 void *data);
 
+hsa_status_t hsa_agent_iterate_caches(hsa_agent_t agent,
+                                hsa_status_t (*callback)(hsa_cache_t cache,
+                                                         void *data),
+                                void *data);
+
+hsa_status_t hsa_agent_iterate_isas(hsa_agent_t agent,
+                                hsa_status_t (*callback)(hsa_isa_t isa,
+                                                         void *data),
+                                void *data);
+
 typedef struct hsa_signal_s {
   uint64_t handle;
 } hsa_signal_t;
 
 #ifdef HSA_LARGE_MODEL
 typedef int64_t hsa_signal_value_t;
 #else
 typedef int32_t hsa_signal_value_t;

openmp/libomptarget/plugins/amdgpu/dynamic_hsa/hsa.cpp

 #include <dlfcn.h>
 
 DLWRAP_INITIALIZE();
 
 DLWRAP_INTERNAL(hsa_init, 0);
 
 DLWRAP(hsa_status_string, 2);
 DLWRAP(hsa_shut_down, 0);
+DLWRAP(hsa_system_get_info, 2);
 DLWRAP(hsa_agent_get_info, 3);
+DLWRAP(hsa_cache_get_info, 3);
+DLWRAP(hsa_isa_get_info_alt, 3);
 DLWRAP(hsa_iterate_agents, 2);
+DLWRAP(hsa_agent_iterate_caches, 3);
+DLWRAP(hsa_agent_iterate_isas, 3);
 DLWRAP(hsa_signal_create, 4);
 DLWRAP(hsa_signal_destroy, 1);
 DLWRAP(hsa_signal_store_relaxed, 2);
 DLWRAP(hsa_signal_store_screlease, 2);
 DLWRAP(hsa_signal_wait_scacquire, 5);
 DLWRAP(hsa_queue_create, 8);
 DLWRAP(hsa_queue_destroy, 1);
 DLWRAP(hsa_queue_load_read_index_scacquire, 1);

openmp/libomptarget/plugins/amdgpu/dynamic_hsa/hsa_ext_amd.h

 
 typedef enum hsa_amd_memory_pool_global_flag_s {
   HSA_AMD_MEMORY_POOL_GLOBAL_FLAG_KERNARG_INIT = 1,
   HSA_AMD_MEMORY_POOL_GLOBAL_FLAG_FINE_GRAINED = 2,
   HSA_AMD_MEMORY_POOL_GLOBAL_FLAG_COARSE_GRAINED = 4
 } hsa_amd_memory_pool_global_flag_t;
 
 typedef enum {
+  HSA_AMD_SEGMENT_GLOBAL = 0,
+  HSA_AMD_SEGMENT_READONLY = 1,
+  HSA_AMD_SEGMENT_PRIVATE = 2,
+  HSA_AMD_SEGMENT_GROUP = 3,
+} hsa_amd_segment_t;
+
+typedef enum {
+  HSA_AMD_MEMORY_POOL_INFO_SEGMENT = 0,
   HSA_AMD_MEMORY_POOL_INFO_GLOBAL_FLAGS = 1,
   HSA_AMD_MEMORY_POOL_INFO_SIZE = 2,
   HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_ALLOWED = 5,
+  HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_GRANULE = 6,
+  HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_ALIGNMENT = 7,
+  HSA_AMD_MEMORY_POOL_INFO_ACCESSIBLE_BY_ALL = 15,
+  HSA_AMD_MEMORY_POOL_INFO_ALLOC_MAX_SIZE = 16,
 } hsa_amd_memory_pool_info_t;
 
 typedef enum {
   HSA_AMD_AGENT_MEMORY_POOL_INFO_ACCESS = 0,
 } hsa_amd_agent_memory_pool_info_t;
 
 typedef enum {
   HSA_AMD_MEMORY_POOL_ACCESS_NEVER_ALLOWED = 0,
 } hsa_amd_memory_pool_access_t;
 
 typedef enum hsa_amd_agent_info_s {
+  HSA_AMD_AGENT_INFO_CACHELINE_SIZE = 0xA001,
   HSA_AMD_AGENT_INFO_COMPUTE_UNIT_COUNT = 0xA002,
+  HSA_AMD_AGENT_INFO_MAX_CLOCK_FREQUENCY = 0xA003,
+  HSA_AMD_AGENT_INFO_PRODUCT_NAME = 0xA009,
+  HSA_AMD_AGENT_INFO_MAX_WAVES_PER_CU = 0xA00A,
+  HSA_AMD_AGENT_INFO_NUM_SIMDS_PER_CU = 0xA00B,
+  HSA_AMD_AGENT_INFO_COOPERATIVE_QUEUES = 0xA010
 } hsa_amd_agent_info_t;
 
 hsa_status_t hsa_amd_memory_pool_get_info(hsa_amd_memory_pool_t memory_pool,
                                           hsa_amd_memory_pool_info_t attribute,
                                           void *value);
 
 hsa_status_t hsa_amd_agent_iterate_memory_pools(
     hsa_agent_t agent,

openmp/libomptarget/plugins/amdgpu/src/rtl.cpp

if (status != HSA_STATUS_SUCCESS) {
     DP("[%s:%d] GPU error in queue %p %d (%s)\n", __FILE__, __LINE__, source,
        status, status_string);
     abort();
   }
 }
 
 namespace core {
 namespace {
+
+bool checkResult(hsa_status_t Err, const char *ErrMsg) {
+  if (Err == HSA_STATUS_SUCCESS)
+    return true;
+
+  REPORT("%s", ErrMsg);
+  REPORT("%s", get_error_string(Err));
+  return false;
+}
+
 void packet_store_release(uint32_t *packet, uint16_t header, uint16_t rest) {
   __atomic_store_n(packet, header | (rest << 16), __ATOMIC_RELEASE);
 }
 
 uint16_t create_header() {
   uint16_t header = HSA_PACKET_TYPE_KERNEL_DISPATCH << HSA_PACKET_HEADER_TYPE;
   header |= HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_ACQUIRE_FENCE_SCOPE;
   header |= HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_RELEASE_FENCE_SCOPE;
hsa_status_t freesignalpool_memcpy_d2h(void *dest, void *src, size_t size,
     return freesignalpool_memcpy(dest, src, size, impl_memcpy_d2h, deviceId);
   }
 
   hsa_status_t freesignalpool_memcpy_h2d(void *dest, void *src, size_t size,
                                          int32_t deviceId) {
     return freesignalpool_memcpy(dest, src, size, impl_memcpy_h2d, deviceId);
   }
 
+  static void printDeviceInfo(int32_t device_id, hsa_agent_t agent) {
+    char TmpChar[1000];
+    uint16_t major, minor;
+    uint32_t TmpUInt;
+    uint32_t TmpUInt2;
+    uint32_t CacheSize[4];
+    bool TmpBool;
+    uint16_t workgroupMaxDim[3];
+    hsa_dim3_t gridMaxDim;
+
+    // Getting basic information about HSA and Device
+    core::checkResult(hsa_system_get_info(HSA_SYSTEM_INFO_VERSION_MAJOR, &major),
+                "Error from hsa_system_get_info when obtaining HSA_SYSTEM_INFO_VERSION_MAJOR\n");
+    core::checkResult(hsa_system_get_info(HSA_SYSTEM_INFO_VERSION_MINOR, &minor),
+                "Error from hsa_system_get_info when obtaining HSA_SYSTEM_INFO_VERSION_MINOR\n");
+    printf("    HSA Runtime Version: \t\t%u.%u \n", major, minor);
+    printf("    HSA OpenMP Device Number: \t\t%d \n", device_id);
+    core::checkResult(hsa_agent_get_info(agent, (hsa_agent_info_t)HSA_AMD_AGENT_INFO_PRODUCT_NAME, TmpChar),
+                "Error returned from hsa_agent_get_info when obtaining HSA_AMD_AGENT_INFO_PRODUCT_NAME\n");
+    printf("    Product Name: \t\t\t%s \n", TmpChar);
+    core::checkResult(hsa_agent_get_info(agent, HSA_AGENT_INFO_NAME, TmpChar),
+                "Error returned from hsa_agent_get_info when obtaining HSA_AGENT_INFO_NAME\n");
+    printf("    Device Name: \t\t\t%s \n", TmpChar);
+    core::checkResult(hsa_agent_get_info(agent, HSA_AGENT_INFO_VENDOR_NAME, TmpChar),
+                "Error returned from hsa_agent_get_info when obtaining HSA_AGENT_INFO_NAME\n");
+    printf("    Vendor Name: \t\t\t%s \n", TmpChar);
+    hsa_device_type_t devType;
+    core::checkResult(hsa_agent_get_info(agent, HSA_AGENT_INFO_DEVICE, &devType),
+                "Error returned from hsa_agent_get_info when obtaining HSA_AGENT_INFO_DEVICE\n");
+    printf("    Device Type: \t\t\t%s \n", devType == HSA_DEVICE_TYPE_CPU ? "CPU": 
+                                           (devType == HSA_DEVICE_TYPE_GPU ? "GPU":
+                                            (devType == HSA_DEVICE_TYPE_DSP ? "DSP": "UNKNOWN")));
+    core::checkResult(hsa_agent_get_info(agent, HSA_AGENT_INFO_QUEUES_MAX, &TmpUInt),
+                "Error returned from hsa_agent_get_info when obtaining HSA_AGENT_INFO_QUEUES_MAX\n");
+    printf("    Max Queues: \t\t\t%u \n", TmpUInt);
+    core::checkResult(hsa_agent_get_info(agent, HSA_AGENT_INFO_QUEUE_MIN_SIZE, &TmpUInt),
+                "Error returned from hsa_agent_get_info when obtaining HSA_AGENT_INFO_QUEUE_MIN_SIZE\n");
+    printf("    Queue Min Size: \t\t\t%u \n", TmpUInt);
+    core::checkResult(hsa_agent_get_info(agent, HSA_AGENT_INFO_QUEUE_MAX_SIZE, &TmpUInt),
+                "Error returned from hsa_agent_get_info when obtaining HSA_AGENT_INFO_QUEUE_MAX_SIZE\n");
+    printf("    Queue Max Size: \t\t\t%u \n", TmpUInt);
+
+    // Getting cache information
+    printf("    Cache:\n");
+
+    // FIXME: This is the right way to get the cache size information according to HSA 1.2.
+    // However, this causes a segmentation fault. So for now we use the deprecated form 
+    // auto CB_cache = [] (const hsa_cache_t cache, void* data) -> hsa_status_t {
+    //   char TmpChar[1000];
+    //   uint8_t level;
+    //   uint32_t size;
+    //   uint32_t name_len;
+
+    //   core::checkResult(hsa_cache_get_info(cache, HSA_CACHE_INFO_NAME_LENGTH, &name_len),
+    //             "Error returned from hsa_cache_get_info when obtaining HSA_CACHE_INFO_NAME_LENGTH\n");
+    //   core::checkResult(hsa_cache_get_info(cache, HSA_CACHE_INFO_NAME, TmpChar),
+    //             "Error returned from hsa_cache_get_info when obtaining HSA_CACHE_INFO_NAME\n");
+    //   core::checkResult(hsa_cache_get_info(cache, HSA_CACHE_INFO_LEVEL, &level),
+    //             "Error returned from hsa_cache_get_info when obtaining HSA_CACHE_INFO_LEVEL\n");
+    //   core::checkResult(hsa_cache_get_info(cache, HSA_CACHE_INFO_SIZE, &size),
+    //             "Error returned from hsa_cache_get_info when obtaining HSA_CACHE_INFO_SIZE\n");
+    //   if (name_len != 0)
+    //     printf("      L%u (%s): \t\t\t%u \n", level, TmpChar, size);
+    //   else
+    //     printf("      L%u: \t\t\t%u \n", level, size);
+    // };
+    // core::checkResult(hsa_agent_iterate_caches(agent, CB_cache, nullptr),
+    //         "Error returned from hsa_agent_iterate_caches\n");
+
+  
+    // FIXME: This is deprecated according to HSA documentation. But the right way is broken
+    core::checkResult(hsa_agent_get_info(agent, HSA_AGENT_INFO_CACHE_SIZE, CacheSize),
+          "Error returned from hsa_agent_get_info when obtaining HSA_AGENT_INFO_CACHE_SIZE\n");
+
+    for (int i = 0; i < 4; i++) {
+      if (CacheSize[i]) {
+        printf("      L%u: \t\t\t\t%u bytes\n", i, CacheSize[i]);
+      }
+    }
+
+    core::checkResult(hsa_agent_get_info(agent, (hsa_agent_info_t) HSA_AMD_AGENT_INFO_CACHELINE_SIZE, &TmpUInt),
+                "Error returned from hsa_agent_get_info when obtaining HSA_AMD_AGENT_INFO_CACHELINE_SIZE\n");
+    printf("    Cacheline Size: \t\t\t%u \n", TmpUInt);
+    core::checkResult(hsa_agent_get_info(agent, (hsa_agent_info_t) HSA_AMD_AGENT_INFO_MAX_CLOCK_FREQUENCY, &TmpUInt),
+                "Error returned from hsa_agent_get_info when obtaining HSA_AMD_AGENT_INFO_MAX_CLOCK_FREQUENCY\n");
+    printf("    Max Clock Freq(MHz): \t\t%u \n", TmpUInt);
+    core::checkResult(hsa_agent_get_info(agent, (hsa_agent_info_t) HSA_AMD_AGENT_INFO_COMPUTE_UNIT_COUNT, &TmpUInt),
+                "Error returned from hsa_agent_get_info when obtaining HSA_AMD_AGENT_INFO_COMPUTE_UNIT_COUNT\n");
+    printf("    Compute Units: \t\t\t%u \n", TmpUInt);
+    core::checkResult(hsa_agent_get_info(agent, (hsa_agent_info_t) HSA_AMD_AGENT_INFO_NUM_SIMDS_PER_CU, &TmpUInt),
+                "Error returned from hsa_agent_get_info when obtaining HSA_AMD_AGENT_INFO_NUM_SIMDS_PER_CU\n");
+    printf("    SIMD per CU: \t\t\t%u \n", TmpUInt);
+    core::checkResult(hsa_agent_get_info(agent, HSA_AGENT_INFO_FAST_F16_OPERATION, &TmpBool),
+                "Error returned from hsa_agent_get_info when obtaining HSA_AMD_AGENT_INFO_NUM_SIMDS_PER_CU\n");
+    printf("    Fast F16 Operation: \t\t%s \n", (TmpBool? "TRUE": "FALSE"));
+    core::checkResult(hsa_agent_get_info(agent,   HSA_AGENT_INFO_WAVEFRONT_SIZE, &TmpUInt2),
+                "Error returned from hsa_agent_get_info when obtaining HSA_AGENT_INFO_WAVEFRONT_SIZE\n");
+    printf("    Wavefront Size: \t\t\t%u \n", TmpUInt2);
+    core::checkResult(hsa_agent_get_info(agent, HSA_AGENT_INFO_WORKGROUP_MAX_SIZE, &TmpUInt),
+                "Error returned from hsa_agent_get_info when obtaining HSA_AGENT_INFO_WORKGROUP_MAX_SIZE\n");
+    printf("    Workgroup Max Size: \t\t%u \n", TmpUInt);
+    core::checkResult(hsa_agent_get_info(agent, HSA_AGENT_INFO_WORKGROUP_MAX_DIM, workgroupMaxDim),
+                "Error returned from hsa_agent_get_info when obtaining HSA_AGENT_INFO_WORKGROUP_MAX_DIM\n");
+    printf("    Workgroup Max Size per Dimension:\n");
+    printf("      x: \t\t\t\t%u\n",workgroupMaxDim[0]);
+    printf("      y: \t\t\t\t%u\n",workgroupMaxDim[1]);
+    printf("      z: \t\t\t\t%u\n",workgroupMaxDim[2]);
+    core::checkResult(hsa_agent_get_info(agent, (hsa_agent_info_t)HSA_AMD_AGENT_INFO_MAX_WAVES_PER_CU, &TmpUInt),
+                "Error returned from hsa_agent_get_info when obtaining HSA_AMD_AGENT_INFO_MAX_WAVES_PER_CU\n");
+    printf("    Max Waves Per CU: \t\t\t%u \n", TmpUInt);
+    printf("    Max Work-item Per CU: \t\t%u \n", TmpUInt*TmpUInt2);
+    core::checkResult(hsa_agent_get_info(agent, HSA_AGENT_INFO_GRID_MAX_SIZE, &TmpUInt),
+                "Error returned from hsa_agent_get_info when obtaining HSA_AGENT_INFO_GRID_MAX_SIZE\n");
+    printf("    Grid Max Size: \t\t\t%u \n", TmpUInt);
+    core::checkResult(hsa_agent_get_info(agent, HSA_AGENT_INFO_GRID_MAX_DIM, &gridMaxDim),
+                "Error returned from hsa_agent_get_info when obtaining HSA_AGENT_INFO_GRID_MAX_DIM\n");
+    printf("    Grid Max Size per Dimension: \t\t\n");
+    printf("      x: \t\t\t\t%u\n",gridMaxDim.x);
+    printf("      y: \t\t\t\t%u\n",gridMaxDim.y);
+    printf("      z: \t\t\t\t%u\n",gridMaxDim.z);
+    core::checkResult(hsa_agent_get_info(agent, HSA_AGENT_INFO_FBARRIER_MAX_SIZE, &TmpUInt),
+                "Error returned from hsa_agent_get_info when obtaining HSA_AGENT_INFO_FBARRIER_MAX_SIZE\n");
+    printf("    Max fbarriers/Workgrp: \t\t%u\n",TmpUInt);
+
+
+    printf("    Memory Pools:\n");
+    auto CB_mem = [] (hsa_amd_memory_pool_t region, void *data) -> hsa_status_t {
+      std::string TmpStr;
+      size_t size;
+      bool alloc, access;
+      hsa_amd_segment_t segment;
+      hsa_amd_memory_pool_global_flag_t globalFlags;
+      core::checkResult(hsa_amd_memory_pool_get_info(region, HSA_AMD_MEMORY_POOL_INFO_GLOBAL_FLAGS, &globalFlags),
+                "Error returned from hsa_amd_memory_pool_get_info when obtaining HSA_AMD_MEMORY_POOL_INFO_GLOBAL_FLAGS\n");
+      core::checkResult(hsa_amd_memory_pool_get_info(region, HSA_AMD_MEMORY_POOL_INFO_SEGMENT, &segment),
+                "Error returned from hsa_amd_memory_pool_get_info when obtaining HSA_AMD_MEMORY_POOL_INFO_SEGMENT\n");
+      
+      switch (segment) {
+        case HSA_AMD_SEGMENT_GLOBAL:
+          TmpStr = "GLOBAL; FLAGS: ";
+          if (HSA_AMD_MEMORY_POOL_GLOBAL_FLAG_KERNARG_INIT & globalFlags)
+            TmpStr += "KERNARG, ";
+          if (HSA_AMD_MEMORY_POOL_GLOBAL_FLAG_FINE_GRAINED & globalFlags)
+            TmpStr += "FINE GRAINED, ";
+          if (HSA_AMD_MEMORY_POOL_GLOBAL_FLAG_COARSE_GRAINED & globalFlags)
+            TmpStr += "COARSE GRAINED, ";
+          break;
+        case HSA_AMD_SEGMENT_READONLY:
+          TmpStr = "READONLY";
+          break;
+        case HSA_AMD_SEGMENT_PRIVATE:
+          TmpStr = "PRIVATE";
+          break;
+        case HSA_AMD_SEGMENT_GROUP:
+          TmpStr = "GROUP";
+          break;
+        default:
+          TmpStr = "unknown";
+          break; 
+      }
+      printf("      Pool %s: \n", TmpStr.c_str());
+
+      core::checkResult(hsa_amd_memory_pool_get_info(region, HSA_AMD_MEMORY_POOL_INFO_SIZE, &size),
+            "Error returned from hsa_amd_memory_pool_get_info when obtaining HSA_AMD_MEMORY_POOL_INFO_SIZE\n");
+      printf("        Size: \t\t\t\t %zu bytes\n", size);
+      core::checkResult(hsa_amd_memory_pool_get_info(region, HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_ALLOWED, &alloc),
+            "Error returned from hsa_amd_memory_pool_get_info when obtaining HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_ALLOWED\n");
+      printf("        Allocatable: \t\t\t %s\n", (alloc ? "TRUE" : "FALSE") );
+      core::checkResult(hsa_amd_memory_pool_get_info(region, HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_GRANULE, &size),
+            "Error returned from hsa_amd_memory_pool_get_info when obtaining HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_GRANULE\n");
+      printf("        Runtime Alloc Granule: \t\t %zu bytes\n", size );
+      core::checkResult(hsa_amd_memory_pool_get_info(region, HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_ALIGNMENT, &size),
+            "Error returned from hsa_amd_memory_pool_get_info when obtaining HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_ALIGNMENT\n");
+      printf("        Runtime Alloc alignment: \t %zu bytes\n", size );
+      core::checkResult(hsa_amd_memory_pool_get_info(region, HSA_AMD_MEMORY_POOL_INFO_ACCESSIBLE_BY_ALL, &access),
+            "Error returned from hsa_amd_memory_pool_get_info when obtaining HSA_AMD_MEMORY_POOL_INFO_ACCESSIBLE_BY_ALL\n");
+      printf("        Accessable by all: \t\t %s\n",  (access ? "TRUE" : "FALSE") );
+
+      return HSA_STATUS_SUCCESS;
+
+    };
+    // Iterate over all the memory regions for this agent. Get the memory region type and size
+    hsa_amd_agent_iterate_memory_pools(agent, CB_mem , nullptr);
+
+    printf("    ISAs:\n");
+    auto CB_isas = [] (hsa_isa_t isa, void *data) -> hsa_status_t {
+      char TmpChar[1000];
+      core::checkResult(hsa_isa_get_info_alt(isa, HSA_ISA_INFO_NAME, TmpChar),
+            "Error returned from hsa_isa_get_info_alt when obtaining HSA_ISA_INFO_NAME\n");
+      printf("        Name: \t\t\t\t %s\n", TmpChar);
+      
+      return HSA_STATUS_SUCCESS;
+
+    };
+    // Iterate over all the memory regions for this agent. Get the memory region type and size
+    hsa_agent_iterate_isas(agent, CB_isas , nullptr);
+  
+  }
+
   // Record entry point associated with device
   void addOffloadEntry(int32_t device_id, __tgt_offload_entry entry) {
     assert(device_id < (int32_t)FuncGblEntries.size() &&
            "Unexpected device id!");
     FuncOrGblEntryTy &E = FuncGblEntries[device_id].back();
 
     E.Entries.push_back(entry);
   }
int32_t __tgt_rtl_synchronize(int32_t device_id, __tgt_async_info *AsyncInfo) {
   // Cuda asserts that AsyncInfo->Queue is non-null, but this invariant
   // is not ensured by devices.cpp for amdgcn
   // assert(AsyncInfo->Queue && "AsyncInfo->Queue is nullptr");
   if (AsyncInfo->Queue) {
     finiAsyncInfo(AsyncInfo);
   }
   return OFFLOAD_SUCCESS;
 }
+
+void __tgt_rtl_print_device_info(int32_t device_id) {
+  // TODO: Assertion to see if device_id is correct
+  // NOTE: We don't need to set context for print device info.
+  
+  DeviceInfo.printDeviceInfo(device_id, DeviceInfo.HSAAgents[device_id]);
+}
+
 } // extern "C"

openmp/libomptarget/plugins/generic-elf-64bit/src/rtl.cpp

DP("Entries table range is (" DPxMOD ")->(" DPxMOD ")\n",
      DPxPTR(entries_begin), DPxPTR(entries_end));
   DeviceInfo.createOffloadTable(device_id, entries_begin, entries_end);
 
   elf_end(e);
 
   return DeviceInfo.getOffloadEntriesTable(device_id);
 }
 
+void __tgt_rtl_print_device_info(int32_t device_id) {
+  printf("    This is a generic-elf-64bit device\n");
+}
+
 // Sample implementation of explicit memory allocator. For this plugin all kinds
 // are equivalent to each other.
 void *__tgt_rtl_data_alloc(int32_t device_id, int64_t size, void *hst_ptr,
                            int32_t kind) {
   void *ptr = NULL;
 
   switch (kind) {
   case TARGET_ALLOC_DEVICE: