[AMDGPU][Libomptarget] Delete g_atmi_initialized

[pdhaliwal]

Summary

This patch drops g_atmi_initialized and inlines the Initialize &
Finalize methods from Runtime class.

[pdhaliwal]

openmp/libomptarget/plugins/amdgpu/impl/machine.cpp

16

This was redundant.

openmp/libomptarget/plugins/amdgpu/impl/rt.h

57–62

This needs to move to D102692.

openmp/libomptarget/plugins/amdgpu/impl/system.cpp

186

This check is always true.

[JonChesterfield]

openmp/libomptarget/plugins/amdgpu/impl/rt.h

57–62

looks like a separate thing - can we reduce this patch to deleting the variable and the dead branches on it, and shuffle code around + inline things in a separate one?

[pdhaliwal]

Removed inlining from this patch and rebased.

[JonChesterfield]

Great, thanks!

openmp/libomptarget/plugins/amdgpu/impl/atmi.cpp

 /*===--------------------------------------------------------------------------
  *              ATMI (Asynchronous Task and Memory Interface)
  *
  * This file is distributed under the MIT License. See LICENSE.txt for details.
  *===------------------------------------------------------------------------*/
 #include "atmi_runtime.h"
 #include "internal.h"
 #include "rt.h"
 #include <hsa.h>
 #include <hsa_ext_amd.h>
 #include <memory>
 
 /*
- * Initialize/Finalize
- */
-atmi_status_t atmi_init() { return core::Runtime::Initialize(); }
-
-atmi_status_t atmi_finalize() { return core::Runtime::Finalize(); }
-
-/*
  * Machine Info
  */
 atmi_machine_t *atmi_machine_get_info() {
   return core::Runtime::GetMachineInfo();
 }
 
 /*
  * Data

openmp/libomptarget/plugins/amdgpu/impl/atmi_interop_hsa.cpp

 /*===--------------------------------------------------------------------------
  *              ATMI (Asynchronous Task and Memory Interface)
  *
  * This file is distributed under the MIT License. See LICENSE.txt for details.
  *===------------------------------------------------------------------------*/
 #include "internal.h"
 
-using core::atl_is_atmi_initialized;
-
 atmi_status_t atmi_interop_hsa_get_symbol_info(
     std::map<std::string, atl_symbol_info_t> &SymbolInfoTable,
     atmi_mem_place_t place, const char *symbol, void **var_addr,
     unsigned int *var_size) {
   /*
      // Typical usage:
      void *var_addr;
      size_t var_size;
      atmi_interop_hsa_get_symbol_addr(gpu_place, "symbol_name", &var_addr,
      &var_size);
      atmi_memcpy(signal, host_add, var_addr, var_size);
   */
 
-  if (!atl_is_atmi_initialized())
-    return ATMI_STATUS_ERROR;
   atmi_machine_t *machine = atmi_machine_get_info();
   if (!symbol || !var_addr || !var_size || !machine)
     return ATMI_STATUS_ERROR;
   if (place.dev_id < 0 ||
       place.dev_id >= machine->device_count_by_type[place.dev_type])
     return ATMI_STATUS_ERROR;
 
   // get the symbol info
atmi_status_t atmi_interop_hsa_get_kernel_info(
   /*
      // Typical usage:
      uint32_t value;
      atmi_interop_hsa_get_kernel_addr(gpu_place, "kernel_name",
                                   HSA_EXECUTABLE_SYMBOL_INFO_KERNEL_KERNARG_SEGMENT_SIZE,
                                   &val);
   */
 
-  if (!atl_is_atmi_initialized())
-    return ATMI_STATUS_ERROR;
   atmi_machine_t *machine = atmi_machine_get_info();
   if (!kernel_name || !value || !machine)
     return ATMI_STATUS_ERROR;
   if (place.dev_id < 0 ||
       place.dev_id >= machine->device_count_by_type[place.dev_type])
     return ATMI_STATUS_ERROR;
 
   atmi_status_t status = ATMI_STATUS_SUCCESS;

openmp/libomptarget/plugins/amdgpu/impl/machine.cpp

 /*===--------------------------------------------------------------------------
  *              ATMI (Asynchronous Task and Memory Interface)
  *
  * This file is distributed under the MIT License. See LICENSE.txt for details.
  *===------------------------------------------------------------------------*/
 #include "machine.h"
 #include "atmi_runtime.h"
 #include "internal.h"
 #include <cassert>
 #include <hsa.h>
 #include <hsa_ext_amd.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <vector>
 
-extern ATLMachine g_atl_machine;
-
 void ATLProcessor::addMemory(const ATLMemory &mem) {
   for (auto &mem_obj : memories_) {
     // if the memory already exists, then just return
     if (mem.memory().handle == mem_obj.memory().handle)
       return;
   }
   memories_.push_back(mem);
 }

openmp/libomptarget/plugins/amdgpu/impl/rt.h

 
 class Runtime final {
 public:
   static Runtime &getInstance() {
     static Runtime instance;
     return instance;
   }
 
-  // init/finalize
-  static atmi_status_t Initialize();
-  static atmi_status_t Finalize();
   // machine info
   static atmi_machine_t *GetMachineInfo();
-  // modules
-  static atmi_status_t RegisterModuleFromMemory(
-      void *, size_t, atmi_place_t,
-      atmi_status_t (*on_deserialized_data)(void *data, size_t size,
-                                            void *cb_state),
-      void *cb_state, std::vector<hsa_executable_t> &HSAExecutables);
 
   // data
   static atmi_status_t Memcpy(hsa_signal_t, void *, const void *, size_t);
   static atmi_status_t Memfree(void *);
   static atmi_status_t Malloc(void **, size_t, atmi_mem_place_t);
 
   int getMaxQueueSize() const { return env_.getMaxQueueSize(); }
   int getDebugMode() const { return env_.getDebugMode(); }

openmp/libomptarget/plugins/amdgpu/impl/system.cpp

 };
 
 // global variables. TODO: Get rid of these
 atmi_machine_t g_atmi_machine;
 ATLMachine g_atl_machine;
 
 std::vector<hsa_amd_memory_pool_t> atl_gpu_kernarg_pools;
 
-bool g_atmi_initialized = false;
 bool g_atmi_hostcall_required = false;
 
 /*
    atlc is all internal global values.
    The structure atl_context_t is defined in atl_internal.h
    Most references will use the global structure prefix atlc.
 */
 atl_context_t atlc = {.struct_initialized = false};
 
 namespace core {
 /* Machine Info */
 atmi_machine_t *Runtime::GetMachineInfo() {
   if (!atlc.g_hsa_initialized)
     return NULL;
   return &g_atmi_machine;
 }
 
-static void atl_set_atmi_initialized() {
-  // FIXME: thread safe? locks?
-  g_atmi_initialized = true;
-}
-
-static void atl_reset_atmi_initialized() {
-  // FIXME: thread safe? locks?
-  g_atmi_initialized = false;
-}
-
-bool atl_is_atmi_initialized() { return g_atmi_initialized; }
-
 void allow_access_to_all_gpu_agents(void *ptr) {
   hsa_status_t err;
   std::vector<ATLGPUProcessor> &gpu_procs =
       g_atl_machine.processors<ATLGPUProcessor>();
   std::vector<hsa_agent_t> agents;
   for (uint32_t i = 0; i < gpu_procs.size(); i++) {
     agents.push_back(gpu_procs[i].agent());
   }
   err = hsa_amd_agents_allow_access(agents.size(), &agents[0], NULL, ptr);
   if (err != HSA_STATUS_SUCCESS) {
     printf("[%s:%d] %s failed: %s\n", __FILE__, __LINE__,
            "Allow agents ptr access", get_error_string(err));
     exit(1);
   }
 }
 
-atmi_status_t Runtime::Initialize() {
-  atmi_devtype_t devtype = ATMI_DEVTYPE_GPU;
-  if (atl_is_atmi_initialized())
-    return ATMI_STATUS_SUCCESS;
-
-  if (devtype == ATMI_DEVTYPE_ALL || devtype & ATMI_DEVTYPE_GPU) {
-    if (atl_init_gpu_context() != ATMI_STATUS_SUCCESS) {
-      printf("[%s:%d] %s failed: %s\n", __FILE__, __LINE__, "GPU context init",
-             get_atmi_error_string(atl_init_gpu_context()));
-      exit(1);
-    }
-  }
-
-  atl_set_atmi_initialized();
-  return ATMI_STATUS_SUCCESS;
-}
-
-atmi_status_t Runtime::Finalize() {
-  hsa_status_t err;
-
-  atl_reset_atmi_initialized();
-  err = hsa_shut_down();
-  if (err != HSA_STATUS_SUCCESS) {
-    printf("[%s:%d] %s failed: %s\n", __FILE__, __LINE__, "Shutting down HSA",
-           get_error_string(err));
-    exit(1);
-  }
-
-  return ATMI_STATUS_SUCCESS;
-}
-
 static void atmi_init_context_structs() {
   atlc.struct_initialized = true; /* This only gets called one time */
   atlc.g_hsa_initialized = false;
   atlc.g_gpu_initialized = false;
   atlc.g_tasks_initialized = false;
 }
 
 // Implement memory_pool iteration function

openmp/libomptarget/plugins/amdgpu/src/rtl.cpp

RTLDeviceInfoTy() {
     //  1 => tracing dispatch only
     // >1 => verbosity increase
     if (char *envStr = getenv("LIBOMPTARGET_KERNEL_TRACE"))
       print_kernel_trace = atoi(envStr);
     else
       print_kernel_trace = 0;
 
     DP("Start initializing HSA-ATMI\n");
-    atmi_status_t err = atmi_init();
+    atmi_status_t err = core::atl_init_gpu_context();
     if (err != ATMI_STATUS_SUCCESS) {
       DP("Error when initializing HSA-ATMI\n");
       return;
     }
+
     // Init hostcall soon after initializing ATMI
     hostrpc_init();
 
     HSAAgents = find_gpu_agents();
     NumberOfDevices = (int)HSAAgents.size();
 
     if (NumberOfDevices == 0) {
       DP("There are no devices supporting HSA.\n");
~RTLDeviceInfoTy() {
     deviceStateStore.clear();
     KernelArgPoolMap.clear();
     KernelInfoTable.clear();
     SymbolInfoTable.clear();
     // Terminate hostrpc before finalizing ATMI
     hostrpc_terminate();
 
     DestroyHSAExecutables();
-    atmi_finalize();
+    hsa_status_t Err = hsa_shut_down();
+    if (Err != HSA_STATUS_SUCCESS) {
+      printf("[%s:%d] %s failed: %s\n", __FILE__, __LINE__, "Shutting down HSA",
+             get_error_string(Err));
+      return;
+    }
   }
 };
 
 pthread_mutex_t SignalPoolT::mutex = PTHREAD_MUTEX_INITIALIZER;
 
 // TODO: May need to drop the trailing to fields until deviceRTL is updated
 struct omptarget_device_environmentTy {
   int32_t debug_level; // gets value of envvar LIBOMPTARGET_DEVICE_RTL_DEBUG