[openmp][libomptarget] Introduce and call createAsyncInfo

[JonChesterfield]

Summary

At present, every async function is responsible for noticing it has
been passed a nullptr for async_info and initializing it. With this change,
a new api function is called to initialize async_info within libomptarget.

A subsequent change can then only call async functions when that construction
succeeded, at which point the init-if-passed-nullptr logic from the plugins
can be replaced with asserts.

Couple of typo fixes noticed while implementing. Builds on D96438.

[JonChesterfield]

• clang-format

openmp/libomptarget/include/omptargetplugin.h

76

API is redundant at present, but it matches the other functions and we might want to return more detailed information than success/fail later

openmp/libomptarget/plugins/cuda/src/rtl.cpp

470–471

moved this to make it public

[JonChesterfield]

• delete debugging cruft

[carlo.bertolli]

LGTM

[JonChesterfield]

• rebase

[ye-luo]

I never liked init-if-passed-nullptr and prefer this explicit approach.

openmp/libomptarget/plugins/amdgpu/src/rtl.cpp

2207

Should the name be __tgt_rtl_create_async_info?

openmp/libomptarget/plugins/cuda/src/rtl.cpp

944

getStream should remain private. It leaks CUstream if put public.
getStream doesn't recreate AsyncInfo->Queue if it is not nullptr.
I would expect an error if createAsyncInfo.
So Please add a new public function instead of reusing getStream.

1453

Should the name be __tgt_rtl_create_async_info?

openmp/libomptarget/src/device.cpp

456

I'm wondering how is !AsyncInfo is compiled. AsyncInfoTy doesn't seem to have "operator bool"

[jdoerfert]

openmp/libomptarget/plugins/cuda/src/rtl.cpp

944

I think my asyncmalloc patch makes it public too.

[ye-luo]

openmp/libomptarget/plugins/cuda/src/rtl.cpp

944

I think my asyncmalloc patch makes it public too.

Then my concern applies to your patch as well. LOL

[JonChesterfield]

openmp/libomptarget/plugins/amdgpu/src/rtl.cpp

2207

yeah, it should. ouch. trying to think of a way to verify things like that (another patch had the wrong prototype, which meant it defined an unrelated function with c++ mangling that got ignored)

openmp/libomptarget/plugins/cuda/src/rtl.cpp

944

seems reasonable to keep the CUstream hidden, will do so

openmp/libomptarget/src/device.cpp

456

Interesting yeah, a brief trial with godbolt shows both gcc and clang rejecting similar constructs. Quirk of compiler flags in use perhaps. I'm assuming it was compiled to 'true' here.

openmp/libomptarget/include/omptarget.h

class AsyncInfoTy {
   /// Locations we used in (potentially) asynchronous calls which should live
   /// as long as this AsyncInfoTy object.
   std::deque<void *> BufferLocations;
 
   __tgt_async_info AsyncInfo;
   DeviceTy &Device;
 
 public:
-  AsyncInfoTy(DeviceTy &Device) : Device(Device) {}
+  AsyncInfoTy(DeviceTy &Device);
   ~AsyncInfoTy() { synchronize(); }
 
   /// Implicit conversion to the __tgt_async_info which is used in the
   /// plugin interface.
   operator __tgt_async_info *() { return &AsyncInfo; }
 
   /// Synchronize all pending actions.
   ///

openmp/libomptarget/include/omptargetplugin.h

 // initialize the target data mapping structures. These addresses are
 // used to generate a table of target variables to pass to
 // __tgt_rtl_run_region(). The __tgt_rtl_data_alloc() returns NULL in
 // case an error occurred on the target device. Kind dictates what allocator
 // to use (e.g. shared, host, device).
 void *__tgt_rtl_data_alloc(int32_t ID, int64_t Size, void *HostPtr,
                            int32_t Kind);
 
+// Create object for device synchronization. In case of success, return zero
+// and AsyncInfo->Queue is not a nullptr. On failure, return an error code and
+// set AsyncInfo->Queue to nullptr.
+int32_t __tgt_rtl_create_async_info(int32_t ID, __tgt_async_info *AsyncInfo);
+
 // Pass the data content to the target device using the target address. In case
 // of success, return zero. Otherwise, return an error code.
 int32_t __tgt_rtl_data_submit(int32_t ID, void *TargetPtr, void *HostPtr,
                               int64_t Size);
 
 int32_t __tgt_rtl_data_submit_async(int32_t ID, void *TargetPtr, void *HostPtr,
                                     int64_t Size, __tgt_async_info *AsyncInfo);
 

openmp/libomptarget/plugins/amdgpu/src/rtl.cpp

void *__tgt_rtl_data_alloc(int device_id, int64_t size, void *, int32_t kind) {
   hsa_amd_memory_pool_t MemoryPool = DeviceInfo.getDeviceMemoryPool(device_id);
   hsa_status_t err = hsa_amd_memory_pool_allocate(MemoryPool, size, 0, &ptr);
   DP("Tgt alloc data %ld bytes, (tgt:%016llx).\n", size,
      (long long unsigned)(Elf64_Addr)ptr);
   ptr = (err == HSA_STATUS_SUCCESS) ? ptr : NULL;
   return ptr;
 }
 
+int32_t __tgt_rtl_create_data_async(int, __tgt_async_info *AsyncInfo) {
+  initAsyncInfo(AsyncInfo);
+  return OFFLOAD_SUCCESS;
+}
+
 int32_t __tgt_rtl_data_submit(int device_id, void *tgt_ptr, void *hst_ptr,
                               int64_t size) {
   assert(device_id < DeviceInfo.NumberOfDevices && "Device ID too large");
   __tgt_async_info AsyncInfo;
   int32_t rc = dataSubmit(device_id, tgt_ptr, hst_ptr, size, &AsyncInfo);
   if (rc != OFFLOAD_SUCCESS)
     return OFFLOAD_FAIL;
 

openmp/libomptarget/plugins/cuda/src/rtl.cpp

class DeviceRTLTy {
 
   // Clear entries table for a device
   void clearOffloadEntriesTable(const int DeviceId) {
     DeviceData[DeviceId].FuncGblEntries.emplace_back();
     FuncOrGblEntryTy &E = DeviceData[DeviceId].FuncGblEntries.back();
     E.Entries.clear();
     E.Table.EntriesBegin = E.Table.EntriesEnd = nullptr;
   }
 
-  CUstream getStream(const int DeviceId, __tgt_async_info *AsyncInfo) const {
-    assert(AsyncInfo && "AsyncInfo is nullptr");
-
-    if (!AsyncInfo->Queue)
-      AsyncInfo->Queue = StreamManager->getStream(DeviceId);
-
-    return reinterpret_cast<CUstream>(AsyncInfo->Queue);
-  }
-
 public:
   // This class should not be copied
   DeviceRTLTy(const DeviceRTLTy &) = delete;
   DeviceRTLTy(DeviceRTLTy &&) = delete;
 
   DeviceRTLTy()
       : NumberOfDevices(0), EnvNumTeams(-1), EnvTeamLimit(-1),
         EnvTeamThreadLimit(-1), RequiresFlags(OMP_REQ_UNDEFINED),
         DynamicMemorySize(0) {
void *dataAlloc(const int DeviceId, const int64_t Size,
     }
 
     REPORT("Invalid target data allocation kind or requested allocator not "
            "implemented yet\n");
 
     return nullptr;
   }
 
+  CUstream getStream(const int DeviceId, __tgt_async_info *AsyncInfo) const {
+    assert(AsyncInfo && "AsyncInfo is nullptr");
+
+    if (!AsyncInfo->Queue)
+      AsyncInfo->Queue = StreamManager->getStream(DeviceId);
+
+    return reinterpret_cast<CUstream>(AsyncInfo->Queue);
+  }
+
   int dataSubmit(const int DeviceId, const void *TgtPtr, const void *HstPtr,
                  const int64_t Size, __tgt_async_info *AsyncInfo) const {
     assert(AsyncInfo && "AsyncInfo is nullptr");
 
     CUresult Err = cuCtxSetCurrent(DeviceData[DeviceId].Context);
     if (!checkResult(Err, "Error returned from cuCtxSetCurrent\n"))
       return OFFLOAD_FAIL;
 

 
 void *__tgt_rtl_data_alloc(int32_t device_id, int64_t size, void *,
                            int32_t kind) {
   assert(DeviceRTL.isValidDeviceId(device_id) && "device_id is invalid");
 
   return DeviceRTL.dataAlloc(device_id, size, (TargetAllocTy)kind);
 }
 
+int32_t __tgt_rtl_create_data_async(int DeviceId, __tgt_async_info *AsyncInfo) {
+  DeviceRTL.getStream(DeviceId, AsyncInfo);
+  return AsyncInfo->Queue ? OFFLOAD_SUCCESS : OFFLOAD_FAIL;
+}
+
 int32_t __tgt_rtl_data_submit(int32_t device_id, void *tgt_ptr, void *hst_ptr,
                               int64_t size) {
   assert(DeviceRTL.isValidDeviceId(device_id) && "device_id is invalid");
 
   __tgt_async_info AsyncInfo;
   const int32_t rc = __tgt_rtl_data_submit_async(device_id, tgt_ptr, hst_ptr,
                                                  size, &AsyncInfo);
   if (rc != OFFLOAD_SUCCESS)

openmp/libomptarget/plugins/exports

 VERS1.0 {
   global:
     __tgt_rtl_is_valid_binary;
     __tgt_rtl_is_data_exchangable;
     __tgt_rtl_number_of_devices;
     __tgt_rtl_init_requires;
     __tgt_rtl_init_device;
     __tgt_rtl_load_binary;
     __tgt_rtl_data_alloc;
+    __tgt_rtl_create_async_info;
     __tgt_rtl_data_submit;
     __tgt_rtl_data_submit_async;
     __tgt_rtl_data_retrieve;
     __tgt_rtl_data_retrieve_async;
     __tgt_rtl_data_exchange;
     __tgt_rtl_data_exchange_async;
     __tgt_rtl_data_delete;
     __tgt_rtl_run_target_team_region;

openmp/libomptarget/src/device.h

struct DeviceTy {
   /// implementations ignore \p HstPtr. \p Kind dictates what allocator should
   /// be used (host, shared, device).
   void *allocData(int64_t Size, void *HstPtr = nullptr,
                   int32_t Kind = TARGET_ALLOC_DEFAULT);
   /// Deallocates memory which \p TgtPtrBegin points at and returns
   /// OFFLOAD_SUCCESS/OFFLOAD_FAIL when succeeds/fails.
   int32_t deleteData(void *TgtPtrBegin);
 
+  /// Create async info, OFFLOAD_SUCCESS/OFFLOAD_FAIL when succeeds/fails
+  /// and sets the internal Queue to !nullptr/nullptr for success/fail
+  int32_t createAsyncInfo(AsyncInfoTy &AsyncInfo);
+
   // Data transfer. When AsyncInfo is nullptr, the transfer will be
   // synchronous.
   // Copy data from host to device
   int32_t submitData(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size,
                      AsyncInfoTy &AsyncInfo);
   // Copy data from device back to host
   int32_t retrieveData(void *HstPtrBegin, void *TgtPtrBegin, int64_t Size,
                        AsyncInfoTy &AsyncInfo);

openmp/libomptarget/src/device.cpp

 void *DeviceTy::allocData(int64_t Size, void *HstPtr, int32_t Kind) {
   return RTL->data_alloc(RTLDeviceID, Size, HstPtr, Kind);
 }
 
 int32_t DeviceTy::deleteData(void *TgtPtrBegin) {
   return RTL->data_delete(RTLDeviceID, TgtPtrBegin);
 }
 
+int32_t DeviceTy::createAsyncInfo(AsyncInfoTy &AsyncInfo) {
+  if (RTL->create_async_info)
+    return RTL->create_async_info(RTLDeviceID, AsyncInfo);
+  return OFFLOAD_FAIL;
+}
+
 // Submit data to device
 int32_t DeviceTy::submitData(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size,
                              AsyncInfoTy &AsyncInfo) {
   if (getInfoLevel() & OMP_INFOTYPE_DATA_TRANSFER) {
     LookupResult LR = lookupMapping(HstPtrBegin, Size);
     auto *HT = &*LR.Entry;
 
     INFO(OMP_INFOTYPE_DATA_TRANSFER, DeviceID,
          "Copying data from host to device, HstPtr=" DPxMOD ", TgtPtr=" DPxMOD
          ", Size=%" PRId64 ", Name=%s\n",
          DPxPTR(HstPtrBegin), DPxPTR(TgtPtrBegin), Size,
          (HT && HT->HstPtrName) ? getNameFromMapping(HT->HstPtrName).c_str()
                                 : "unknown");
   }
 
-  if (!AsyncInfo || !RTL->data_submit_async || !RTL->synchronize)
+  if (!RTL->data_submit_async || !RTL->synchronize)
     return RTL->data_submit(RTLDeviceID, TgtPtrBegin, HstPtrBegin, Size);
   else
     return RTL->data_submit_async(RTLDeviceID, TgtPtrBegin, HstPtrBegin, Size,
                                   AsyncInfo);
 }
 
 // Retrieve data from device
 int32_t DeviceTy::retrieveData(void *HstPtrBegin, void *TgtPtrBegin,
int32_t DeviceTy::retrieveData(void *HstPtrBegin, void *TgtPtrBegin,
   else
     return RTL->data_retrieve_async(RTLDeviceID, HstPtrBegin, TgtPtrBegin, Size,
                                     AsyncInfo);
 }
 
 // Copy data from current device to destination device directly
 int32_t DeviceTy::dataExchange(void *SrcPtr, DeviceTy &DstDev, void *DstPtr,
                                int64_t Size, AsyncInfoTy &AsyncInfo) {
-  if (!AsyncInfo || !RTL->data_exchange_async || !RTL->synchronize) {
+  if (!RTL->data_exchange_async || !RTL->synchronize) {
     assert(RTL->data_exchange && "RTL->data_exchange is nullptr");
     return RTL->data_exchange(RTLDeviceID, SrcPtr, DstDev.RTLDeviceID, DstPtr,
                               Size);
   } else
     return RTL->data_exchange_async(RTLDeviceID, SrcPtr, DstDev.RTLDeviceID,
                                     DstPtr, Size, AsyncInfo);
 }
 
 // Run region on device
 int32_t DeviceTy::runRegion(void *TgtEntryPtr, void **TgtVarsPtr,
                             ptrdiff_t *TgtOffsets, int32_t TgtVarsSize,
                             AsyncInfoTy &AsyncInfo) {
-  if (!RTL->run_region || !RTL->synchronize)
+  if (!RTL->run_region_async || !RTL->synchronize)
     return RTL->run_region(RTLDeviceID, TgtEntryPtr, TgtVarsPtr, TgtOffsets,
                            TgtVarsSize);
   else
     return RTL->run_region_async(RTLDeviceID, TgtEntryPtr, TgtVarsPtr,
                                  TgtOffsets, TgtVarsSize, AsyncInfo);
 }
 
 // Run region on device

openmp/libomptarget/src/omptarget.cpp

 #include "omptarget.h"
 #include "device.h"
 #include "private.h"
 #include "rtl.h"
 
 #include <cassert>
 #include <vector>
 
+AsyncInfoTy::AsyncInfoTy(DeviceTy &Device) : Device(Device) {
+  int Result = Device.createAsyncInfo(*this);
+  assert(
+      ((Result == OFFLOAD_SUCCESS) && (AsyncInfo.Queue != nullptr) ||
+       (Result != OFFLOAD_SUCCESS) && (AsyncInfo.Queue == nullptr)) &&
+      "The device plugin returned inconsistent return value and queue pointer");
+  (void)Result;
+}
+
 int AsyncInfoTy::synchronize() {
   int Result = OFFLOAD_SUCCESS;
   if (AsyncInfo.Queue) {
     // If we have a queue we need to synchronize it now.
     Result = Device.synchronize(*this);
     assert(AsyncInfo.Queue == nullptr &&
            "The device plugin should have nulled the queue to indicate there "
            "are no outstanding actions!");

openmp/libomptarget/src/rtl.h

 
 struct RTLInfoTy {
   typedef int32_t(is_valid_binary_ty)(void *);
   typedef int32_t(is_data_exchangable_ty)(int32_t, int32_t);
   typedef int32_t(number_of_devices_ty)();
   typedef int32_t(init_device_ty)(int32_t);
   typedef __tgt_target_table *(load_binary_ty)(int32_t, void *);
   typedef void *(data_alloc_ty)(int32_t, int64_t, void *, int32_t);
+  typedef int32_t(create_async_info_ty)(int32_t, __tgt_async_info *);
   typedef int32_t(data_submit_ty)(int32_t, void *, void *, int64_t);
   typedef int32_t(data_submit_async_ty)(int32_t, void *, void *, int64_t,
                                         __tgt_async_info *);
   typedef int32_t(data_retrieve_ty)(int32_t, void *, void *, int64_t);
   typedef int32_t(data_retrieve_async_ty)(int32_t, void *, void *, int64_t,
                                           __tgt_async_info *);
   typedef int32_t(data_exchange_ty)(int32_t, void *, int32_t, void *, int64_t);
   typedef int32_t(data_exchange_async_ty)(int32_t, void *, int32_t, void *,
#endif
 
   // Functions implemented in the RTL.
   is_valid_binary_ty *is_valid_binary = nullptr;
   is_data_exchangable_ty *is_data_exchangable = nullptr;
   number_of_devices_ty *number_of_devices = nullptr;
   init_device_ty *init_device = nullptr;
   load_binary_ty *load_binary = nullptr;
   data_alloc_ty *data_alloc = nullptr;
+  create_async_info_ty *create_async_info = nullptr;
   data_submit_ty *data_submit = nullptr;
   data_submit_async_ty *data_submit_async = nullptr;
   data_retrieve_ty *data_retrieve = nullptr;
   data_retrieve_async_ty *data_retrieve_async = nullptr;
   data_exchange_ty *data_exchange = nullptr;
   data_exchange_async_ty *data_exchange_async = nullptr;
   data_delete_ty *data_delete = nullptr;
   run_region_ty *run_region = nullptr;

openmp/libomptarget/src/rtl.cpp

 #endif
 
     DP("Registering RTL %s supporting %d devices!\n", R.RTLName.c_str(),
        R.NumberOfDevices);
 
     // Optional functions
     *((void **)&R.init_requires) =
         dlsym(dynlib_handle, "__tgt_rtl_init_requires");
+    *((void **)&R.create_async_info) =
+        dlsym(dynlib_handle, "__tgt_rtl_create_async_info");
     *((void **)&R.data_submit_async) =
         dlsym(dynlib_handle, "__tgt_rtl_data_submit_async");
     *((void **)&R.data_retrieve_async) =
         dlsym(dynlib_handle, "__tgt_rtl_data_retrieve_async");
     *((void **)&R.run_region_async) =
         dlsym(dynlib_handle, "__tgt_rtl_run_target_region_async");
     *((void **)&R.run_team_region_async) =
         dlsym(dynlib_handle, "__tgt_rtl_run_target_team_region_async");