diff --git a/openmp/libomptarget/include/omptargetplugin.h b/openmp/libomptarget/include/omptargetplugin.h --- a/openmp/libomptarget/include/omptargetplugin.h +++ b/openmp/libomptarget/include/omptargetplugin.h @@ -142,6 +142,20 @@ // Set plugin's internal information flag externally. void __tgt_rtl_set_info_flag(uint32_t); +// Create an event at the moment when this function is called based on +// AsyncInfo. The returned event can be used later for setting dependency or +// synchronization. +void *__tgt_rtl_create_event(int32_t ID, __tgt_async_info *AsyncInfo); + +// Destroy the event created by __tgt_rtl_create_event previously. +int32_t __tgt_rtl_destroy_event(int32_t ID, void *Event, + __tgt_async_info *AsyncInfo); + +// Wait for the event. It can be used for setting dependences. Depending on +// targets, it can be blocking or non-blocking. +int32_t __tgt_rtl_wait_event(int32_t ID, void *Event, + __tgt_async_info *AsyncInfo); + #ifdef __cplusplus } #endif diff --git a/openmp/libomptarget/plugins/cuda/src/rtl.cpp b/openmp/libomptarget/plugins/cuda/src/rtl.cpp --- a/openmp/libomptarget/plugins/cuda/src/rtl.cpp +++ b/openmp/libomptarget/plugins/cuda/src/rtl.cpp @@ -127,6 +127,29 @@ return OFFLOAD_SUCCESS; } +void *createEvent(__tgt_async_info *AsyncInfo) { + CUstream Stream = reinterpret_cast(AsyncInfo->Queue); + CUevent Event = nullptr; + + CUresult Err = cuEventCreate(&Event, CU_EVENT_DEFAULT); + if (Err != CUDA_SUCCESS) { + DP("Error when creating event. stream = " DPxMOD ", event = " DPxMOD "\n", + DPxPTR(Stream), DPxPTR(Event)); + CUDA_ERR_STRING(Err); + return nullptr; + } + + Err = cuEventRecord(Event, Stream); + if (Err != CUDA_SUCCESS) { + DP("Error when recording event. stream = " DPxMOD ", event = " DPxMOD "\n", + DPxPTR(Stream), DPxPTR(Event)); + CUDA_ERR_STRING(Err); + return nullptr; + } + + return (void *)Event; +} + // Structure contains per-device data struct DeviceDataTy { /// List that contains all the kernels. @@ -1157,6 +1180,39 @@ } return (Err == CUDA_SUCCESS) ? OFFLOAD_SUCCESS : OFFLOAD_FAIL; } + + int waitEvent(const int DeviceId, __tgt_async_info *AsyncInfo, + void *EventPtr) const { + CUstream Stream = getStream(DeviceId, AsyncInfo); + CUevent Event = reinterpret_cast(EventPtr); + + CUresult Err = cuStreamWaitEvent(Stream, Event, CU_EVENT_WAIT_DEFAULT); + if (Err != CUDA_SUCCESS) { + DP("Error when waiting event. stream = " DPxMOD ", event = " DPxMOD "\n", + DPxPTR(Stream), DPxPTR(Event)); + CUDA_ERR_STRING(Err); + return OFFLOAD_FAIL; + } + + return OFFLOAD_SUCCESS; + } + + int destroyEvent(const int DeviceId, __tgt_async_info *AsyncInfo, + void *EventPtr) const { + CUstream Stream = reinterpret_cast(AsyncInfo->Queue); + CUevent Event = reinterpret_cast(EventPtr); + + CUresult Err = cuEventDestroy(Event); + if (Err != CUDA_SUCCESS) { + DP("Error when destroying event. stream = " DPxMOD ", event = " DPxMOD + "\n", + DPxPTR(Stream), DPxPTR(Event)); + CUDA_ERR_STRING(Err); + return OFFLOAD_FAIL; + } + + return OFFLOAD_SUCCESS; + } }; DeviceRTLTy DeviceRTL; @@ -1357,6 +1413,34 @@ InfoLevel.store(NewInfoLevel); } +void *__tgt_rtl_create_event(int32_t device_id, + __tgt_async_info *async_info_ptr) { + assert(DeviceRTL.isValidDeviceId(device_id) && "device_id is invalid"); + assert(async_info_ptr && "async_info_ptr is nullptr"); + assert(async_info_ptr->Queue && "async_info_ptr->Queue is nullptr"); + + return createEvent(async_info_ptr); +} + +int32_t __tgt_rtl_wait_event(int32_t device_id, void *event_ptr, + __tgt_async_info *async_info_ptr) { + assert(DeviceRTL.isValidDeviceId(device_id) && "device_id is invalid"); + assert(async_info_ptr && "async_info_ptr is nullptr"); + assert(event_ptr && "event is nullptr"); + + return DeviceRTL.waitEvent(device_id, async_info_ptr, event_ptr); +} + +int32_t __tgt_rtl_destroy_event(int32_t device_id, void *event_ptr, + __tgt_async_info *async_info_ptr) { + assert(DeviceRTL.isValidDeviceId(device_id) && "device_id is invalid"); + assert(async_info_ptr && "async_info_ptr is nullptr"); + assert(async_info_ptr->Queue && "async_info_ptr->Queue is nullptr"); + assert(event_ptr && "event is nullptr"); + + return DeviceRTL.destroyEvent(device_id, async_info_ptr, event_ptr); +} + #ifdef __cplusplus } #endif diff --git a/openmp/libomptarget/plugins/exports b/openmp/libomptarget/plugins/exports --- a/openmp/libomptarget/plugins/exports +++ b/openmp/libomptarget/plugins/exports @@ -23,6 +23,9 @@ __tgt_rtl_unregister_lib; __tgt_rtl_supports_empty_images; __tgt_rtl_set_info_flag; + __tgt_rtl_create_event; + __tgt_rtl_destroy_event; + __tgt_rtl_wait_event; local: *; }; diff --git a/openmp/libomptarget/src/device.h b/openmp/libomptarget/src/device.h --- a/openmp/libomptarget/src/device.h +++ b/openmp/libomptarget/src/device.h @@ -49,6 +49,9 @@ uintptr_t TgtPtrBegin; // target info. + /// Pointer to the event corresponding to the data update of this map. + mutable void *Event; + private: /// use mutable to allow modification via std::set iterator which is const. mutable uint64_t RefCount; @@ -65,7 +68,7 @@ HostDataToTargetTy(uintptr_t BP, uintptr_t B, uintptr_t E, uintptr_t TB, map_var_info_t Name = nullptr, bool IsINF = false) : HstPtrBase(BP), HstPtrBegin(B), HstPtrEnd(E), HstPtrName(Name), - TgtPtrBegin(TB), RefCount(IsINF ? INFRefCount : 1), + TgtPtrBegin(TB), Event(nullptr), RefCount(IsINF ? INFRefCount : 1), UpdateMtx(std::make_shared()) {} uint64_t getRefCount() const { return RefCount; } @@ -275,6 +278,12 @@ /// OFFLOAD_SUCCESS/OFFLOAD_FAIL when succeeds/fails. int32_t synchronize(AsyncInfoTy &AsyncInfo); + void *createEvent(AsyncInfoTy &AsyncInfo); + + int32_t destroyEvent(void *Event, AsyncInfoTy &AsyncInfo); + + int32_t waitEvent(void *Event, AsyncInfoTy &AsyncInfo); + private: // Call to RTL void init(); // To be called only via DeviceTy::initOnce() diff --git a/openmp/libomptarget/src/device.cpp b/openmp/libomptarget/src/device.cpp --- a/openmp/libomptarget/src/device.cpp +++ b/openmp/libomptarget/src/device.cpp @@ -258,29 +258,63 @@ if (IsNew && MoveData == MoveDataStateTy::UNKNOWN) MoveData = MoveDataStateTy::REQUIRED; - // If the target pointer is valid, and we need to transfer data, issue the - // data transfer. - if (TargetPointer && (MoveData == MoveDataStateTy::REQUIRED)) { - // Lock the entry before releasing the mapping table lock such that another - // thread that could issue data movement will get the right result. - Entry->lock(); - // Release the mapping table lock right after the entry is locked. - DataMapMtx.unlock(); + if (TargetPointer) { + // If the target pointer is valid, and we need to transfer data, issue the + // data transfer. + if (MoveData == MoveDataStateTy::REQUIRED) { + // Lock the entry before releasing the mapping table lock such that + // another thread that could issue data movement will get the right + // result. + Entry->lock(); + // Release the mapping table lock right after the entry is locked. + DataMapMtx.unlock(); - DP("Moving %" PRId64 " bytes (hst:" DPxMOD ") -> (tgt:" DPxMOD ")\n", Size, - DPxPTR(HstPtrBegin), DPxPTR(TargetPointer)); + DP("Moving %" PRId64 " bytes (hst:" DPxMOD ") -> (tgt:" DPxMOD ")\n", + Size, DPxPTR(HstPtrBegin), DPxPTR(TargetPointer)); - int Ret = submitData(TargetPointer, HstPtrBegin, Size, AsyncInfo); + int Ret = submitData(TargetPointer, HstPtrBegin, Size, AsyncInfo); - // Unlock the entry immediately after the data movement is issued. - Entry->unlock(); + if (Ret != OFFLOAD_SUCCESS) { + // Unlock the entry immediately if data movement issuing reports error. + Entry->unlock(); - if (Ret != OFFLOAD_SUCCESS) { - REPORT("Copying data to device failed.\n"); - // We will also return nullptr if the data movement fails because that - // pointer points to a corrupted memory region so it doesn't make any - // sense to continue to use it. - TargetPointer = nullptr; + REPORT("Copying data to device failed.\n"); + // We will also return nullptr if the data movement fails because that + // pointer points to a corrupted memory region so it doesn't make any + // sense to continue to use it. + TargetPointer = nullptr; + } + + // Create an event at this moment and attach it to the entry. + void *Event = createEvent(AsyncInfo); + void *OldEvent = Entry->Event; + Entry->Event = Event; + // We're done with the entry. Release the entry. + Entry->unlock(); + // If there is an event attached, destroy it. + if (OldEvent) + destroyEvent(OldEvent, AsyncInfo); + } else { + // Release the mapping table lock directly. + DataMapMtx.unlock(); + // If not a host pointer, we need to wait for the event if it exists. + if (!IsHostPtr) { + Entry->lock(); + void *Event = Entry->Event; + Entry->unlock(); + + if (Event) { + int Ret = waitEvent(Event, AsyncInfo); + if (Ret != OFFLOAD_SUCCESS) { + // If it fails to wait for the event, we need to return nullptr in + // case of any data race. + REPORT("Failed to wait for event " DPxMOD ".\n", DPxPTR(Event)); + return {{false /* IsNewEntry */, false /* IsHostPointer */}, + {} /* MapTableEntry */, + nullptr /* TargetPointer */}; + } + } + } } } else { // Release the mapping table lock directly. @@ -544,6 +578,27 @@ return OFFLOAD_SUCCESS; } +void *DeviceTy::createEvent(AsyncInfoTy &AsyncInfo) { + if (RTL->create_event) + return RTL->create_event(RTLDeviceID, AsyncInfo); + + return nullptr; +} + +int32_t DeviceTy::destroyEvent(void *Event, AsyncInfoTy &AsyncInfo) { + if (RTL->create_event) + return RTL->destroy_event(RTLDeviceID, Event, AsyncInfo); + + return OFFLOAD_SUCCESS; +} + +int32_t DeviceTy::waitEvent(void *Event, AsyncInfoTy &AsyncInfo) { + if (RTL->create_event) + return RTL->wait_event(RTLDeviceID, Event, AsyncInfo); + + return OFFLOAD_SUCCESS; +} + /// Check whether a device has an associated RTL and initialize it if it's not /// already initialized. bool device_is_ready(int device_num) { diff --git a/openmp/libomptarget/src/omptarget.cpp b/openmp/libomptarget/src/omptarget.cpp --- a/openmp/libomptarget/src/omptarget.cpp +++ b/openmp/libomptarget/src/omptarget.cpp @@ -584,12 +584,20 @@ int rt = Device.submitData(PointerTgtPtrBegin, &TgtPtrBase, sizeof(void *), AsyncInfo); - Pointer_TPR.MapTableEntry->unlock(); - if (rt != OFFLOAD_SUCCESS) { + Pointer_TPR.MapTableEntry->unlock(); REPORT("Copying data to device failed.\n"); return OFFLOAD_FAIL; } + // Create a new event for this moment + void *Event = Device.createEvent(AsyncInfo); + // Exchange the old event with new created event + void *OldEvent = Pointer_TPR.MapTableEntry->Event; + Pointer_TPR.MapTableEntry->Event = Event; + Pointer_TPR.MapTableEntry->unlock(); + // If the old event is not null, we need to destroy it. + if (OldEvent) + Device.destroyEvent(OldEvent, AsyncInfo); } else Device.ShadowMtx.unlock(); } diff --git a/openmp/libomptarget/src/rtl.h b/openmp/libomptarget/src/rtl.h --- a/openmp/libomptarget/src/rtl.h +++ b/openmp/libomptarget/src/rtl.h @@ -56,6 +56,9 @@ typedef int32_t (*register_lib_ty)(__tgt_bin_desc *); typedef int32_t(supports_empty_images_ty)(); typedef void(set_info_flag_ty)(uint32_t); + typedef void *(create_event_ty)(int32_t, __tgt_async_info *); + typedef int32_t(destroy_event_ty)(int32_t, void *, __tgt_async_info *); + typedef int32_t(wait_event_ty)(int32_t, void *, __tgt_async_info *); int32_t Idx = -1; // RTL index, index is the number of devices // of other RTLs that were registered before, @@ -93,6 +96,9 @@ register_lib_ty unregister_lib = nullptr; supports_empty_images_ty *supports_empty_images = nullptr; set_info_flag_ty *set_info_flag = nullptr; + create_event_ty *create_event = nullptr; + destroy_event_ty *destroy_event = nullptr; + wait_event_ty *wait_event = nullptr; // Are there images associated with this RTL. bool isUsed = false; diff --git a/openmp/libomptarget/src/rtl.cpp b/openmp/libomptarget/src/rtl.cpp --- a/openmp/libomptarget/src/rtl.cpp +++ b/openmp/libomptarget/src/rtl.cpp @@ -177,6 +177,11 @@ dlsym(dynlib_handle, "__tgt_rtl_supports_empty_images"); *((void **)&R.set_info_flag) = dlsym(dynlib_handle, "__tgt_rtl_set_info_flag"); + *((void **)&R.create_event) = + dlsym(dynlib_handle, "__tgt_rtl_create_event"); + *((void **)&R.destroy_event) = + dlsym(dynlib_handle, "__tgt_rtl_destroy_event"); + *((void **)&R.wait_event) = dlsym(dynlib_handle, "__tgt_rtl_wait_event"); } DP("RTLs loaded!\n");