[OpenMP][OMPT] Add 'Initialized' flag

[mhalk]

Summary

We observed some overhead and unnecessary debug output.
This can be alleviated by (re-)introduction of a boolean that indicates, if the
OMPT initialization has been performed.

openmp/libomptarget/include/OmptCallback.h

 /// libomptarget -OR- by libomptarget during finalization of OMPT in the plugin.
 /// \p tool_data as provided by the tool
 void finalizeLibrary(ompt_data_t *tool_data);
 
 /// This function will connect the \p initializeLibrary and \p finalizeLibrary
 /// functions to their respective higher layer.
 void connectLibrary();
 
+/// OMPT initialization status; false if initializeLibrary has not been executed
+extern bool Initialized;
+
 } // namespace ompt
 } // namespace target
 } // namespace omp
 } // namespace llvm
 
 #endif // OMPT_SUPPORT
 
 #pragma pop_macro("DEBUG_PREFIX")
 
 #endif // _OMPTCALLBACK_H

openmp/libomptarget/plugins-nextgen/common/OMPT/OmptCallback.cpp

 #include <memory>
 
 #include "Debug.h"
 #include "OmptCallback.h"
 #include "OmptConnector.h"
 
 using namespace llvm::omp::target::ompt;
 
+bool llvm::omp::target::ompt::Initialized = false;
+
 ompt_get_callback_t llvm::omp::target::ompt::lookupCallbackByCode = nullptr;
 ompt_function_lookup_t llvm::omp::target::ompt::lookupCallbackByName = nullptr;
 
 int llvm::omp::target::ompt::initializeLibrary(ompt_function_lookup_t lookup,
                                                int initial_device_num,
                                                ompt_data_t *tool_data) {
   DP("OMPT: Executing initializeLibrary (libomptarget)\n");
 #define bindOmptFunctionName(OmptFunction, DestinationFunction)                \
   if (lookup)                                                                  \
     DestinationFunction = (OmptFunction##_t)lookup(#OmptFunction);             \
   DP("OMPT: initializeLibrary (libomptarget) bound %s=%p\n",                   \
      #DestinationFunction, ((void *)(uint64_t)DestinationFunction));
 
   bindOmptFunctionName(ompt_get_callback, lookupCallbackByCode);
 #undef bindOmptFunctionName
 
   // Store pointer of 'ompt_libomp_target_fn_lookup' for use by the plugin
   lookupCallbackByName = lookup;
 
+  Initialized = true;
+
   return 0;
 }
 
 void llvm::omp::target::ompt::finalizeLibrary(ompt_data_t *tool_data) {
   DP("OMPT: Executing finalizeLibrary (libomptarget)\n");
 }
 
 void llvm::omp::target::ompt::connectLibrary() {

openmp/libomptarget/plugins-nextgen/common/PluginInterface/PluginInterface.cpp

: MemoryManager(nullptr), OMP_TeamLimit("OMP_TEAM_LIMIT"),
       OMPX_InitialNumEvents("LIBOMPTARGET_NUM_INITIAL_EVENTS", 32),
       DeviceId(DeviceId), GridValues(OMPGridValues),
       PeerAccesses(NumDevices, PeerAccessState::PENDING), PeerAccessesLock(),
       PinnedAllocs(*this), RPCServer(nullptr) {
 #ifdef OMPT_SUPPORT
   OmptInitialized.store(false);
   // Bind the callbacks to this device's member functions
 #define bindOmptCallback(Name, Type, Code)                                     \
-  if (ompt::lookupCallbackByCode)                                              \
+  if (ompt::Initialized && ompt::lookupCallbackByCode) {                       \
     ompt::lookupCallbackByCode((ompt_callbacks_t)(Code),                       \
                                ((ompt_callback_t *)&(Name##_fn)));             \
-  DP("OMPT: class bound %s=%p\n", #Name, ((void *)(uint64_t)Name##_fn));
+    DP("OMPT: class bound %s=%p\n", #Name, ((void *)(uint64_t)Name##_fn));     \
+  }
 
   FOREACH_OMPT_DEVICE_EVENT(bindOmptCallback);
 #undef bindOmptCallback
 
 #endif
 }
 
 Error GenericDeviceTy::init(GenericPluginTy &Plugin) {
   if (auto Err = initImpl(Plugin))
     return Err;
 
 #ifdef OMPT_SUPPORT
+  if (ompt::Initialized) {
     bool ExpectedStatus = false;
     if (OmptInitialized.compare_exchange_strong(ExpectedStatus, true))
       performOmptCallback(device_initialize,
                           /* device_num */ DeviceId,
                           /* type */ getComputeUnitKind().c_str(),
                           /* device */ reinterpret_cast<ompt_device_t *>(this),
                           /* lookup */ ompt::lookupCallbackByName,
                           /* documentation */ nullptr);
+  }
 #endif
 
   // Read and reinitialize the envars that depend on the device initialization.
   // Notice these two envars may change the stack size and heap size of the
   // device, so they need the device properly initialized.
   auto StackSizeEnvarOrErr = UInt64Envar::create(
       "LIBOMPTARGET_STACK_SIZE",
       [this](uint64_t &V) -> Error { return getDeviceStackSize(V); },
Error GenericDeviceTy::deinit() {
   if (RecordReplay.isRecordingOrReplaying())
     RecordReplay.deinit();
 
   if (RPCServer)
     if (auto Err = RPCServer->deinitDevice(*this))
       return Err;
 
 #ifdef OMPT_SUPPORT
+  if (ompt::Initialized) {
     bool ExpectedStatus = true;
     if (OmptInitialized.compare_exchange_strong(ExpectedStatus, false))
       performOmptCallback(device_finalize, /* device_num */ DeviceId);
+  }
 #endif
 
   return deinitImpl();
 }
 Expected<__tgt_target_table *>
 GenericDeviceTy::loadBinary(GenericPluginTy &Plugin,
                             const __tgt_device_image *InputTgtImage) {
   assert(InputTgtImage && "Expected non-null target image");
GenericDeviceTy::loadBinary(GenericPluginTy &Plugin,
   // Register all offload entries of the image.
   if (auto Err = registerOffloadEntries(*Image))
     return std::move(Err);
 
   if (auto Err = setupRPCServer(Plugin, *Image))
     return std::move(Err);
 
 #ifdef OMPT_SUPPORT
-  size_t Bytes = getPtrDiff(InputTgtImage->ImageEnd, InputTgtImage->ImageStart);
+  if (ompt::Initialized) {
+    size_t Bytes =
+        getPtrDiff(InputTgtImage->ImageEnd, InputTgtImage->ImageStart);
     performOmptCallback(device_load,
                         /* device_num */ DeviceId,
                         /* FileName */ nullptr,
                         /* File Offset */ 0,
                         /* VmaInFile */ nullptr,
                         /* ImgSize */ Bytes,
                         /* HostAddr */ InputTgtImage->ImageStart,
                         /* DeviceAddr */ nullptr,
                         /* FIXME: ModuleId */ 0);
+  }
 #endif
 
   // Return the pointer to the table of entries.
   return Image->getOffloadEntryTable();
 }
 
 Error GenericDeviceTy::setupDeviceEnvironment(GenericPluginTy &Plugin,
                                               DeviceImageTy &Image) {

openmp/libomptarget/src/OmptCallback.cpp

 
 private:
   llvm::SmallVector<ompt_finalize_t> RtlFinalizationFunctions;
 };
 
 /// Object that will maintain the RTL finalizer from the plugin
 LibomptargetRtlFinalizer *LibraryFinalizer = nullptr;
 
+bool llvm::omp::target::ompt::Initialized = false;
+
 ompt_get_callback_t llvm::omp::target::ompt::lookupCallbackByCode = nullptr;
 ompt_function_lookup_t llvm::omp::target::ompt::lookupCallbackByName = nullptr;
 
 int llvm::omp::target::ompt::initializeLibrary(ompt_function_lookup_t lookup,
                                                int initial_device_num,
                                                ompt_data_t *tool_data) {
   DP("Executing initializeLibrary (libomp)\n");
 #define bindOmptFunctionName(OmptFunction, DestinationFunction)                \
#undef bindOmptFunctionName
 
   assert(lookupCallbackByCode && "lookupCallbackByCode should be non-null");
   assert(lookupCallbackByName && "lookupCallbackByName should be non-null");
   assert(LibraryFinalizer == nullptr &&
          "LibraryFinalizer should not be initialized yet");
 
   LibraryFinalizer = new LibomptargetRtlFinalizer();
 
+  Initialized = true;
+
   return 0;
 }
 
 void llvm::omp::target::ompt::finalizeLibrary(ompt_data_t *data) {
   DP("Executing finalizeLibrary (libomp)\n");
   // Before disabling OMPT, call the (plugin) finalizations that were registered
   // with this library
   LibraryFinalizer->finalize();
#undef bindOmptCallback
 
   DP("Exiting connectLibrary (libomp)\n");
 }
 
 extern "C" {
 /// Used for connecting libomptarget with a plugin
 void ompt_libomptarget_connect(ompt_start_tool_result_t *result) {
   DP("Enter ompt_libomptarget_connect\n");
-  if (result && LibraryFinalizer) {
+  if (Initialized && result && LibraryFinalizer) {
     // Cache each fini function, so that they can be invoked on exit
     LibraryFinalizer->registerRtl(result->finalize);
     // Invoke the provided init function with the lookup function maintained
     // in this library so that callbacks maintained by this library are
     // retrieved.
     result->initialize(lookupCallbackByName,
                        /* initial_device_num */ 0, /* tool_data */ nullptr);
   }