[OpenMP] Replace global InfoLevel with a reference to an internal one.

[jhuber6]

Summary

This patch improves the implementation of D100774 by replacing the global
variable introduced with a function that returns a reference to an internal
one. This removes the need to define the variable in every plugin that uses it.

[JonChesterfield]

LGTM, thanks.

Is there anything special about the cuda plugin here? I see it's the only one with an implementation of __tgt_rtl_set_info_flag, but wasn't following along closely enough to see what that's used for.

[jhuber6]

In D101102#2710134, @JonChesterfield wrote:

LGTM, thanks.

Is there anything special about the cuda plugin here? I see it's the only one with an implementation of __tgt_rtl_set_info_flag, but wasn't following along closely enough to see what that's used for.

The purpose of the function is so the host can set the info level in the plugin, since they're separate libraries. I only implemented it for cuda because it's the only one that uses InfoLevel right now and it's optional.

[JonChesterfield]

In D101102#2710146, @jhuber6 wrote:

I only implemented it for cuda because it's the only one that uses InfoLevel right now and it's optional.

Yep, I see that. I haven't looked up what it's used for and was wondering if amdgpu wants the same feature. Will leave that for another day though.

[jhuber6]

In D101102#2710382, @JonChesterfield wrote:

Yep, I see that. I haven't looked up what it's used for and was wondering if amdgpu wants the same feature. Will leave that for another day though.

I'd recommend using it if you want to print anything to the user without requiring debug. The current uses are shown in the docs, https://openmp.llvm.org/docs/design/Runtimes.html#libomptarget-info.

openmp/libomptarget/include/Debug.h

#define GCC_VERSION                                                            \
   (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__)
 
 #if !defined(__clang__) && defined(__GNUC__) && GCC_VERSION < 70100
 #define USED __attribute__((used))
 #else
 #define USED
 #endif
 
-// Interface to the InfoLevel variable defined by each library.
-extern std::atomic<uint32_t> InfoLevel;
-
 // Add __attribute__((used)) to work around a bug in gcc 5/6.
-USED static inline uint32_t getInfoLevel() {
+USED inline std::atomic<uint32_t> &getInfoLevelInternal() {
+  static std::atomic<uint32_t> InfoLevel;
   static std::once_flag Flag{};
   std::call_once(Flag, []() {
     if (char *EnvStr = getenv("LIBOMPTARGET_INFO"))
       InfoLevel.store(std::stoi(EnvStr));
   });
 
-  return InfoLevel.load();
+  return InfoLevel;
 }
 
+USED inline uint32_t getInfoLevel() { return getInfoLevelInternal().load(); }
+
 // Add __attribute__((used)) to work around a bug in gcc 5/6.
 USED static inline uint32_t getDebugLevel() {
   static uint32_t DebugLevel = 0;
   static std::once_flag Flag{};
   std::call_once(Flag, []() {
     if (char *EnvStr = getenv("LIBOMPTARGET_DEBUG"))
       DebugLevel = std::stoi(EnvStr);
   });

openmp/libomptarget/plugins/amdgpu/src/rtl.cpp

int32_t __tgt_rtl_synchronize(int32_t device_id, __tgt_async_info *AsyncInfo) {
   // Cuda asserts that AsyncInfo->Queue is non-null, but this invariant
   // is not ensured by devices.cpp for amdgcn
   // assert(AsyncInfo->Queue && "AsyncInfo->Queue is nullptr");
   if (AsyncInfo->Queue) {
     finiAsyncInfo(AsyncInfo);
   }
   return OFFLOAD_SUCCESS;
 }
-
-// AMDGPU plugin's internal InfoLevel.
-std::atomic<uint32_t> InfoLevel;

openmp/libomptarget/plugins/cuda/src/rtl.cpp

 //===----RTLs/cuda/src/rtl.cpp - Target RTLs Implementation ------- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // RTL for CUDA machine
 //
 //===----------------------------------------------------------------------===//
 
 #include <cassert>
 #include <cstddef>
 #include <cuda.h>
 #include <list>
 #include <memory>
 #include <mutex>
 #include <string>
 #include <vector>
 
 #include "Debug.h"
 #include "omptargetplugin.h"
int32_t __tgt_rtl_synchronize(int32_t device_id,
   assert(DeviceRTL.isValidDeviceId(device_id) && "device_id is invalid");
   assert(async_info_ptr && "async_info_ptr is nullptr");
   assert(async_info_ptr->Queue && "async_info_ptr->Queue is nullptr");
 
   return DeviceRTL.synchronize(device_id, async_info_ptr);
 }
 
 void __tgt_rtl_set_info_flag(uint32_t NewInfoLevel) {
+  std::atomic<uint32_t> &InfoLevel = getInfoLevelInternal();
   InfoLevel.store(NewInfoLevel);
 }
 
 #ifdef __cplusplus
 }
 #endif
-
-// Cuda plugin's internal InfoLevel.
-std::atomic<uint32_t> InfoLevel;

openmp/libomptarget/plugins/generic-elf-64bit/src/rtl.cpp

int32_t __tgt_rtl_run_target_region(int32_t device_id, void *tgt_entry_ptr,
   // use one team and one thread.
   return __tgt_rtl_run_target_team_region(device_id, tgt_entry_ptr, tgt_args,
                                           tgt_offsets, arg_num, 1, 1, 0);
 }
 
 #ifdef __cplusplus
 }
 #endif
-
-// Elf-64 plugin's internal InfoLevel.
-std::atomic<uint32_t> InfoLevel;

openmp/libomptarget/plugins/remote/src/rtl.cpp

return Manager->runTargetTeamRegionAsync(
       DeviceId, TgtEntryPtr, TgtArgs, TgtOffsets, ArgNum, TeamNum, ThreadLimit,
       LoopTripCount, AsyncInfo);
 }
 
 // Exposed library API function
 #ifdef __cplusplus
 }
 #endif
-
-// Remote Offloading interal InfoLevel.
-std::atomic<uint32_t> InfoLevel;

openmp/libomptarget/src/interface.cpp

DP("__kmpc_push_target_tripcount(%" PRId64 ", %" PRIu64 ")\n", device_id,
      loop_tripcount);
   PM->TblMapMtx.lock();
   PM->Devices[device_id].LoopTripCnt.emplace(__kmpc_global_thread_num(NULL),
                                              loop_tripcount);
   PM->TblMapMtx.unlock();
 }
 
 EXTERN void __tgt_set_info_flag(uint32_t NewInfoLevel) {
+  std::atomic<uint32_t> &InfoLevel = getInfoLevelInternal();
   InfoLevel.store(NewInfoLevel);
   for (auto &R : PM->RTLs.AllRTLs) {
     if (R.set_info_flag)
       R.set_info_flag(NewInfoLevel);
   }
 }
-
-// Libomptarget's InfoLevel storage.
-std::atomic<uint32_t> InfoLevel;