diff --git a/openmp/libomptarget/include/omptargetplugin.h b/openmp/libomptarget/include/omptargetplugin.h --- a/openmp/libomptarget/include/omptargetplugin.h +++ b/openmp/libomptarget/include/omptargetplugin.h @@ -142,6 +142,20 @@ // Set plugin's internal information flag externally. void __tgt_rtl_set_info_flag(uint32_t); +// Create an event at the moment when this function is called based on +// AsyncInfo. The returned event can be used later for setting dependency or +// synchronization. +void *__tgt_rtl_create_event(int32_t ID, __tgt_async_info *AsyncInfo); + +// Destroy the event created by __tgt_rtl_create_event previously. +int32_t __tgt_rtl_destroy_event(int32_t ID, void *Event, + __tgt_async_info *AsyncInfo); + +// Wait for the event. It can be used for setting dependences. Depending on +// targets, it can be blocking or non-blocking. +int32_t __tgt_rtl_wait_event(int32_t ID, void *Event, + __tgt_async_info *AsyncInfo); + #ifdef __cplusplus } #endif diff --git a/openmp/libomptarget/plugins/cuda/src/rtl.cpp b/openmp/libomptarget/plugins/cuda/src/rtl.cpp --- a/openmp/libomptarget/plugins/cuda/src/rtl.cpp +++ b/openmp/libomptarget/plugins/cuda/src/rtl.cpp @@ -21,6 +21,7 @@ #include #include "Debug.h" +#include "omptarget.h" #include "omptargetplugin.h" #define TARGET_NAME CUDA @@ -125,6 +126,29 @@ return OFFLOAD_SUCCESS; } +void *createEvent(__tgt_async_info *AsyncInfo) { + CUstream Stream = reinterpret_cast(AsyncInfo->Queue); + CUevent Event = nullptr; + + CUresult Err = cuEventCreate(&Event, CU_EVENT_DEFAULT); + if (Err != CUDA_SUCCESS) { + DP("Error when creating event. stream = " DPxMOD ", event = " DPxMOD "\n", + DPxPTR(Stream), DPxPTR(Event)); + CUDA_ERR_STRING(Err); + return nullptr; + } + + Err = cuEventRecord(Event, Stream); + if (Err != CUDA_SUCCESS) { + DP("Error when recording event. stream = " DPxMOD ", event = " DPxMOD "\n", + DPxPTR(Stream), DPxPTR(Event)); + CUDA_ERR_STRING(Err); + return nullptr; + } + + return (void *)Event; +} + // Structure contains per-device data struct DeviceDataTy { /// List that contains all the kernels. @@ -1103,6 +1127,39 @@ } return (Err == CUDA_SUCCESS) ? OFFLOAD_SUCCESS : OFFLOAD_FAIL; } + + int waitEvent(const int DeviceId, __tgt_async_info *AsyncInfo, + void *EventPtr) const { + CUstream Stream = reinterpret_cast(AsyncInfo->Queue); + CUevent Event = reinterpret_cast(EventPtr); + + CUresult Err = cuStreamWaitEvent(Stream, Event, CU_EVENT_WAIT_DEFAULT); + if (Err != CUDA_SUCCESS) { + DP("Error when waiting event. stream = " DPxMOD ", event = " DPxMOD "\n", + DPxPTR(Stream), DPxPTR(Event)); + CUDA_ERR_STRING(Err); + return OFFLOAD_FAIL; + } + + return OFFLOAD_SUCCESS; + } + + int destroyEvent(const int DeviceId, __tgt_async_info *AsyncInfo, + void *EventPtr) const { + CUstream Stream = reinterpret_cast(AsyncInfo->Queue); + CUevent Event = reinterpret_cast(EventPtr); + + CUresult Err = cuEventDestroy(Event); + if (Err != CUDA_SUCCESS) { + DP("Error when destroying event. stream = " DPxMOD ", event = " DPxMOD + "\n", + DPxPTR(Stream), DPxPTR(Event)); + CUDA_ERR_STRING(Err); + return OFFLOAD_FAIL; + } + + return OFFLOAD_SUCCESS; + } }; DeviceRTLTy DeviceRTL; @@ -1303,6 +1360,35 @@ InfoLevel.store(NewInfoLevel); } +void *__tgt_rtl_create_event(int32_t device_id, + __tgt_async_info *async_info_ptr) { + assert(DeviceRTL.isValidDeviceId(device_id) && "device_id is invalid"); + assert(async_info_ptr && "async_info_ptr is nullptr"); + assert(async_info_ptr->Queue && "async_info_ptr->Queue is nullptr"); + + return createEvent(async_info_ptr); +} + +int32_t __tgt_rtl_wait_event(int32_t device_id, void *event_ptr, + __tgt_async_info *async_info_ptr) { + assert(DeviceRTL.isValidDeviceId(device_id) && "device_id is invalid"); + assert(async_info_ptr && "async_info_ptr is nullptr"); + assert(async_info_ptr->Queue && "async_info_ptr->Queue is nullptr"); + assert(event_ptr && "event is nullptr"); + + return DeviceRTL.waitEvent(device_id, async_info_ptr, event_ptr); +} + +int32_t __tgt_rtl_destroy_event(int32_t device_id, void *event_ptr, + __tgt_async_info *async_info_ptr) { + assert(DeviceRTL.isValidDeviceId(device_id) && "device_id is invalid"); + assert(async_info_ptr && "async_info_ptr is nullptr"); + assert(async_info_ptr->Queue && "async_info_ptr->Queue is nullptr"); + assert(event_ptr && "event is nullptr"); + + return DeviceRTL.destroyEvent(device_id, async_info_ptr, event_ptr); +} + #ifdef __cplusplus } #endif diff --git a/openmp/libomptarget/plugins/exports b/openmp/libomptarget/plugins/exports --- a/openmp/libomptarget/plugins/exports +++ b/openmp/libomptarget/plugins/exports @@ -23,6 +23,9 @@ __tgt_rtl_unregister_lib; __tgt_rtl_supports_empty_images; __tgt_rtl_set_info_flag; + __tgt_rtl_create_event; + __tgt_rtl_destroy_event; + __tgt_rtl_wait_event; local: *; }; diff --git a/openmp/libomptarget/src/device.h b/openmp/libomptarget/src/device.h --- a/openmp/libomptarget/src/device.h +++ b/openmp/libomptarget/src/device.h @@ -13,6 +13,7 @@ #ifndef _OMPTARGET_DEVICE_H #define _OMPTARGET_DEVICE_H +#include #include #include #include @@ -49,6 +50,9 @@ uintptr_t TgtPtrBegin; // target info. + /// Pointer to the event corresponding to the data movement of this map. + mutable std::shared_ptr> Event; + private: /// use mutable to allow modification via std::set iterator which is const. mutable uint64_t RefCount; @@ -58,7 +62,8 @@ HostDataToTargetTy(uintptr_t BP, uintptr_t B, uintptr_t E, uintptr_t TB, map_var_info_t Name = nullptr, bool IsINF = false) : HstPtrBase(BP), HstPtrBegin(B), HstPtrEnd(E), HstPtrName(Name), - TgtPtrBegin(TB), RefCount(IsINF ? INFRefCount : 1) {} + TgtPtrBegin(TB), Event(std::make_shared>(nullptr)), + RefCount(IsINF ? INFRefCount : 1) {} uint64_t getRefCount() const { return RefCount; } @@ -241,6 +246,12 @@ /// OFFLOAD_SUCCESS/OFFLOAD_FAIL when succeeds/fails. int32_t synchronize(AsyncInfoTy &AsyncInfo); + void *createEvent(AsyncInfoTy &AsyncInfo); + + int32_t destroyEvent(void *Event, AsyncInfoTy &AsyncInfo); + + int32_t waitEvent(void *Event, AsyncInfoTy &AsyncInfo); + private: // Call to RTL void init(); // To be called only via DeviceTy::initOnce() diff --git a/openmp/libomptarget/src/device.cpp b/openmp/libomptarget/src/device.cpp --- a/openmp/libomptarget/src/device.cpp +++ b/openmp/libomptarget/src/device.cpp @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #include "device.h" +#include "omptarget.h" #include "private.h" #include "rtl.h" @@ -522,6 +523,27 @@ return OFFLOAD_SUCCESS; } +void *DeviceTy::createEvent(AsyncInfoTy &AsyncInfo) { + if (RTL->create_event) + return RTL->create_event(RTLDeviceID, AsyncInfo); + + return nullptr; +} + +int32_t DeviceTy::destroyEvent(void *Event, AsyncInfoTy &AsyncInfo) { + if (RTL->create_event) + return RTL->destroy_event(RTLDeviceID, Event, AsyncInfo); + + return OFFLOAD_SUCCESS; +} + +int32_t DeviceTy::waitEvent(void *Event, AsyncInfoTy &AsyncInfo) { + if (RTL->create_event) + return RTL->wait_event(RTLDeviceID, Event, AsyncInfo); + + return OFFLOAD_SUCCESS; +} + /// Check whether a device has an associated RTL and initialize it if it's not /// already initialized. bool device_is_ready(int device_num) { diff --git a/openmp/libomptarget/src/omptarget.cpp b/openmp/libomptarget/src/omptarget.cpp --- a/openmp/libomptarget/src/omptarget.cpp +++ b/openmp/libomptarget/src/omptarget.cpp @@ -564,7 +564,36 @@ REPORT("Copying data to device failed.\n"); return OFFLOAD_FAIL; } - // TODO: Attach the event in AsyncInfo to the map table entry if needed + // FIXME: We probably need a new flag to indicate whether the device + // supports async data movement. If no, we don't have to create the + // event. + // Create a new event for this moment + void *Event = Device.createEvent(AsyncInfo); + // Exchange the old event with new created event + void *OldEvent = TPR.MapTableEntry->Event->exchange(Event); + // If the old event is not null, we need to destroy it. + if (OldEvent) { + int Ret = Device.destroyEvent(OldEvent, AsyncInfo); + if (Ret != OFFLOAD_SUCCESS) { + // Do something + return OFFLOAD_FAIL; + } + } + } else if (!IsHostPtr) { + // If the data doesn't need to be moved, we need to gaurantee that the + // corresponding event is enqueued to avoid data race. + // Busy wait while the attached event is still nullptr, which means + // the data movement has not been scheduled yet. + // FIXME: We probably need a new flag to indicate whether the device + // supports async data movement. If no, we don't have to wait for the + // event. + while (TPR.MapTableEntry->Event->load() == nullptr) + ; + int Ret = Device.waitEvent(TPR.MapTableEntry->Event->load(), AsyncInfo); + if (Ret != OFFLOAD_SUCCESS) { + // TODO: do something + return OFFLOAD_FAIL; + } } } @@ -580,7 +609,18 @@ REPORT("Copying data to device failed.\n"); return OFFLOAD_FAIL; } - // TODO: Attach the event in AsyncInfo to the map table entry if needed + // Create a new event for this moment + void *Event = Device.createEvent(AsyncInfo); + // Exchange the old event with new created event + void *OldEvent = Pointer_TPR.MapTableEntry->Event->exchange(Event); + // If the old event is not null, we need to destroy it. + if (OldEvent) { + int Ret = Device.destroyEvent(OldEvent, AsyncInfo); + if (Ret != OFFLOAD_SUCCESS) { + // TODO: do something + return OFFLOAD_FAIL; + } + } // create shadow pointers for this entry Device.ShadowMtx.lock(); Device.ShadowPtrMap[Pointer_HstPtrBegin] = { diff --git a/openmp/libomptarget/src/rtl.h b/openmp/libomptarget/src/rtl.h --- a/openmp/libomptarget/src/rtl.h +++ b/openmp/libomptarget/src/rtl.h @@ -56,6 +56,9 @@ typedef int32_t (*register_lib_ty)(__tgt_bin_desc *); typedef int32_t(supports_empty_images_ty)(); typedef void(set_info_flag_ty)(uint32_t); + typedef void *(create_event_ty)(int32_t, __tgt_async_info *); + typedef int32_t(destroy_event_ty)(int32_t, void *, __tgt_async_info *); + typedef int32_t(wait_event_ty)(int32_t, void *, __tgt_async_info *); int32_t Idx = -1; // RTL index, index is the number of devices // of other RTLs that were registered before, @@ -93,6 +96,9 @@ register_lib_ty unregister_lib = nullptr; supports_empty_images_ty *supports_empty_images = nullptr; set_info_flag_ty *set_info_flag = nullptr; + create_event_ty *create_event = nullptr; + destroy_event_ty *destroy_event = nullptr; + wait_event_ty *wait_event = nullptr; // Are there images associated with this RTL. bool isUsed = false; diff --git a/openmp/libomptarget/src/rtl.cpp b/openmp/libomptarget/src/rtl.cpp --- a/openmp/libomptarget/src/rtl.cpp +++ b/openmp/libomptarget/src/rtl.cpp @@ -177,6 +177,11 @@ dlsym(dynlib_handle, "__tgt_rtl_supports_empty_images"); *((void **)&R.set_info_flag) = dlsym(dynlib_handle, "__tgt_rtl_set_info_flag"); + *((void **)&R.create_event) = + dlsym(dynlib_handle, "__tgt_rtl_create_event"); + *((void **)&R.destroy_event) = + dlsym(dynlib_handle, "__tgt_rtl_destroy_event"); + *((void **)&R.wait_event) = dlsym(dynlib_handle, "__tgt_rtl_wait_event"); } DP("RTLs loaded!\n"); diff --git a/openmp/libomptarget/test/offloading/bug49334.cpp b/openmp/libomptarget/test/offloading/bug49334.cpp --- a/openmp/libomptarget/test/offloading/bug49334.cpp +++ b/openmp/libomptarget/test/offloading/bug49334.cpp @@ -70,8 +70,8 @@ } }; -constexpr const int BS = 256; -constexpr const int N = 1024; +constexpr const int BS = 16; +constexpr const int N = 256; int BlockMatMul_TargetNowait(BlockMatrix &A, BlockMatrix &B, BlockMatrix &C) { #pragma omp parallel