[Libomptarget] Revert changes to AMDGPU plugin destructors

[jhuber6]

Summary

These patches exposed a lot of problems in the AMD toolchain. Rather
than keep it broken we should revert it to its old semi-functional
state. This will prevent us from using device destructors but should
remove some new bugs. In the future this interface should be changed
once these problems are addressed more correctly.

This reverts commit ed0f21811544320f829124efbb6a38ee12eb9155.

This reverts commit 2b7203a35972e98b8521f92d2791043dc539ae88.

Fixes #57536

[jdoerfert]

LG, we will work on proper fixes but unblock ppl in the meantime.

openmp/libomptarget/plugins/amdgpu/src/rtl.cpp

private:
   struct Lock {
     Lock(pthread_mutex_t *M) : M(M) { pthread_mutex_lock(M); }
     ~Lock() { pthread_mutex_unlock(M); }
     pthread_mutex_t *M;
   };
 };
 pthread_mutex_t KernelArgPool::Mutex = PTHREAD_MUTEX_INITIALIZER;
 
+std::unordered_map<std::string /*kernel*/, std::unique_ptr<KernelArgPool>>
+    KernelArgPoolMap;
+
 /// Use a single entity to encode a kernel and a set of flags
 struct KernelTy {
   llvm::omp::OMPTgtExecModeFlags ExecutionMode;
   int16_t ConstWGSize;
   int32_t DeviceId;
   void *CallStackAddr = nullptr;
   const char *Name;
 
   KernelTy(llvm::omp::OMPTgtExecModeFlags ExecutionMode, int16_t ConstWgSize,
            int32_t DeviceId, void *CallStackAddr, const char *Name,
            uint32_t KernargSegmentSize,
-           hsa_amd_memory_pool_t &KernArgMemoryPool,
-           std::unordered_map<std::string, std::unique_ptr<KernelArgPool>>
-               &KernelArgPoolMap)
+           hsa_amd_memory_pool_t &KernArgMemoryPool)
       : ExecutionMode(ExecutionMode), ConstWGSize(ConstWgSize),
         DeviceId(DeviceId), CallStackAddr(CallStackAddr), Name(Name) {
     DP("Construct kernelinfo: ExecMode %d\n", ExecutionMode);
 
     std::string N(Name);
     if (KernelArgPoolMap.find(N) == KernelArgPoolMap.end()) {
       KernelArgPoolMap.insert(
           std::make_pair(N, std::unique_ptr<KernelArgPool>(new KernelArgPool(
                                 KernargSegmentSize, KernArgMemoryPool))));
     }
   }
 };
 
+/// List that contains all the kernels.
+/// FIXME: we may need this to be per device and per library.
+std::list<KernelTy> KernelsList;
+
 template <typename Callback> static hsa_status_t findAgents(Callback CB) {
 
   hsa_status_t Err =
       hsa::iterate_agents([&](hsa_agent_t Agent) -> hsa_status_t {
         hsa_device_type_t DeviceType;
         // get_info fails iff HSA runtime not yet initialized
         hsa_status_t Err =
             hsa_agent_get_info(Agent, HSA_AGENT_INFO_DEVICE, &DeviceType);
public:
   bool ConstructionSucceeded = false;
 
   // load binary populates symbol tables and mutates various global state
   // run uses those symbol tables
   std::shared_timed_mutex LoadRunLock;
 
   int NumberOfDevices = 0;
 
-  /// List that contains all the kernels.
-  /// FIXME: we may need this to be per device and per library.
-  std::list<KernelTy> KernelsList;
-  std::unordered_map<std::string /*kernel*/, std::unique_ptr<KernelArgPool>>
-      KernelArgPoolMap;
-
   // GPU devices
   std::vector<hsa_agent_t> HSAAgents;
   std::vector<HSAQueueScheduler> HSAQueueSchedulers; // one per gpu
 
   // CPUs
   std::vector<hsa_agent_t> CPUAgents;
 
   // Device properties
public:
   }
 
   // Clear entries table for a device
   void clearOffloadEntriesTable(int DeviceId) {
     assert(DeviceId < (int32_t)FuncGblEntries.size() &&
            "Unexpected device id!");
     FuncGblEntries[DeviceId].emplace_back();
     FuncOrGblEntryTy &E = FuncGblEntries[DeviceId].back();
+    // KernelArgPoolMap.clear();
     E.Entries.clear();
     E.Table.EntriesBegin = E.Table.EntriesEnd = 0;
   }
 
   hsa_status_t addDeviceMemoryPool(hsa_amd_memory_pool_t MemoryPool,
                                    unsigned int DeviceId) {
     assert(DeviceId < DeviceFineGrainedMemoryPools.size() && "Error here.");
     uint32_t GlobalFlags = 0;
for (uint32_t I = 0; I < HSAExecutables.size(); I++) {
            "Destroying executable", get_error_string(Err));
       }
     }
   }
 };
 
 pthread_mutex_t SignalPoolT::mutex = PTHREAD_MUTEX_INITIALIZER;
 
-static RTLDeviceInfoTy *DeviceInfoState = nullptr;
-static RTLDeviceInfoTy &DeviceInfo() { return *DeviceInfoState; }
+// Putting accesses to DeviceInfo global behind a function call prior
+// to changing to use init_plugin/deinit_plugin calls
+static RTLDeviceInfoTy DeviceInfoState;
+static RTLDeviceInfoTy &DeviceInfo() { return DeviceInfoState; }
 
 namespace {
 
 int32_t dataRetrieve(int32_t DeviceId, void *HstPtr, void *TgtPtr, int64_t Size,
                      __tgt_async_info *AsyncInfo) {
   assert(AsyncInfo && "AsyncInfo is nullptr");
   assert(DeviceId < DeviceInfo().NumberOfDevices && "Device ID too large");
   // Return success if we are not copying back to host from target.
// Run on the device.
     Packet->kernel_object = KernelInfoEntry.kernel_object;
     Packet->kernarg_address = 0;     // use the block allocator
     Packet->reserved2 = 0;           // impl writes id_ here
     Packet->completion_signal = {0}; // may want a pool of signals
 
     KernelArgPool *ArgPool = nullptr;
     void *KernArg = nullptr;
     {
-      auto It =
-          DeviceInfo().KernelArgPoolMap.find(std::string(KernelInfo->Name));
-      if (It != DeviceInfo().KernelArgPoolMap.end()) {
+      auto It = KernelArgPoolMap.find(std::string(KernelInfo->Name));
+      if (It != KernelArgPoolMap.end()) {
         ArgPool = (It->second).get();
       }
     }
     if (!ArgPool) {
       DP("Warning: No ArgPool for %s on device %d\n", KernelInfo->Name,
          DeviceId);
     }
     {
bool IsImageCompatibleWithEnv(const char *ImgInfo, std::string EnvInfo) {
   //   set to ANY
   DP("Compatible: Target IDs are compatible \t[Image: %s]\t:\t[Environment: "
      "%s]\n",
      ImgTID.data(), EnvTID.data());
   return true;
 }
 
 extern "C" {
-
-int32_t __tgt_rtl_init_plugin() {
-  DeviceInfoState = new RTLDeviceInfoTy;
-  return (DeviceInfoState && DeviceInfoState->ConstructionSucceeded)
-             ? OFFLOAD_SUCCESS
-             : OFFLOAD_FAIL;
-}
-
-int32_t __tgt_rtl_deinit_plugin() {
-  if (DeviceInfoState)
-    delete DeviceInfoState;
-  return OFFLOAD_SUCCESS;
-}
-
 int32_t __tgt_rtl_is_valid_binary(__tgt_device_image *Image) {
   return elfMachineIdIsAmdgcn(Image);
 }
 
 int32_t __tgt_rtl_is_valid_binary_info(__tgt_device_image *image,
                                        __tgt_image_info *info) {
   if (!__tgt_rtl_is_valid_binary(image))
     return false;
for (int32_t DeviceId = 0; DeviceId < NumberOfDevices; ++DeviceId) {
     if (!IsImageCompatibleWithEnv(info->Arch, DeviceInfo().TargetID[DeviceId]))
       return false;
   }
   DP("Image has Target ID compatible with the current environment: %s\n",
      info->Arch);
   return true;
 }
 
+int32_t __tgt_rtl_init_plugin() { return OFFLOAD_SUCCESS; }
+int32_t __tgt_rtl_deinit_plugin() { return OFFLOAD_SUCCESS; }
+
 int __tgt_rtl_number_of_devices() {
   // If the construction failed, no methods are safe to call
   if (DeviceInfo().ConstructionSucceeded) {
     return DeviceInfo().NumberOfDevices;
   }
   DP("AMDGPU plugin construction failed. Zero devices available\n");
   return 0;
 }
for (__tgt_offload_entry *E = HostBegin; E != HostEnd; ++E) {
     } else {
       DP("Loading global exec_mode '%s' - symbol missing, using default "
          "value "
          "GENERIC (1)\n",
          ExecModeName);
     }
     check("Loading computation property", Err);
 
-    DeviceInfo().KernelsList.push_back(
-        KernelTy(ExecModeVal, WGSizeVal, DeviceId, CallStackAddr, E->name,
-                 KernargSegmentSize, DeviceInfo().KernArgPool,
-                 DeviceInfo().KernelArgPoolMap));
+    KernelsList.push_back(KernelTy(ExecModeVal, WGSizeVal, DeviceId,
+                                   CallStackAddr, E->name, KernargSegmentSize,
+                                   DeviceInfo().KernArgPool));
     __tgt_offload_entry Entry = *E;
-    Entry.addr = (void *)&DeviceInfo().KernelsList.back();
+    Entry.addr = (void *)&KernelsList.back();
     DeviceInfo().addOffloadEntry(DeviceId, Entry);
     DP("Entry point %ld maps to %s\n", E - HostBegin, E->name);
   }
 
   return DeviceInfo().getOffloadEntriesTable(DeviceId);
 }
 
 void *__tgt_rtl_data_alloc(int DeviceId, int64_t Size, void *, int32_t Kind) {

openmp/libomptarget/src/rtl.cpp

__attribute__((constructor(101))) void init() {
   ProfileTraceFile = getenv("LIBOMPTARGET_PROFILE");
   // TODO: add a configuration option for time granularity
   if (ProfileTraceFile)
     timeTraceProfilerInitialize(500 /* us */, "libomptarget");
 }
 
 __attribute__((destructor(101))) void deinit() {
   DP("Deinit target library!\n");
-
-  for (auto *R : PM->RTLs.UsedRTLs) {
-    // Plugins can either destroy their local state using global variables
-    // or attribute(destructor) functions or by implementing deinit_plugin
-    // The hazard with plugin local destructors is they may be called before
-    // or after this destructor. If the plugin is destroyed using global
-    // state before this library finishes calling into it the plugin is
-    // likely to crash. If good fortune means the plugin outlives this
-    // library then there may be no crash.
-    // Using deinit_plugin and no global destructors from the plugin works.
-    if (R->deinit_plugin) {
-      if (R->deinit_plugin() != OFFLOAD_SUCCESS) {
-        DP("Failure deinitializing RTL %s!\n", R->RTLName.c_str());
-      }
-    }
-  }
-
   delete PM;
 
   if (ProfileTraceFile) {
     // TODO: add env var for file output
     if (auto E = timeTraceProfilerWrite(ProfileTraceFile, "-"))
       fprintf(stderr, "Error writing out the time trace\n");
 
     timeTraceProfilerCleanup();
if (TransTable != PM->HostEntriesBeginToTransTable.end()) {
     DP("Translation table for descriptor " DPxMOD " cannot be found, probably "
        "it has been already removed.\n",
        DPxPTR(Desc->HostEntriesBegin));
   }
 
   PM->TblMapMtx.unlock();
 
   // TODO: Write some RTL->unload_image(...) function?
+  for (auto *R : UsedRTLs) {
+    if (R->deinit_plugin) {
+      if (R->deinit_plugin() != OFFLOAD_SUCCESS) {
+        DP("Failure deinitializing RTL %s!\n", R->RTLName.c_str());
+      }
+    }
+  }
 
   DP("Done unregistering library!\n");
 }