diff --git a/openmp/libomptarget/include/omptargetplugin.h b/openmp/libomptarget/include/omptargetplugin.h --- a/openmp/libomptarget/include/omptargetplugin.h +++ b/openmp/libomptarget/include/omptargetplugin.h @@ -142,6 +142,20 @@ // Set plugin's internal information flag externally. void __tgt_rtl_set_info_flag(uint32_t); +// Create an event at the moment when this function is called based on +// AsyncInfo. The returned event can be used later for setting dependency or +// synchronization. +void *__tgt_rtl_create_event(int32_t ID, __tgt_async_info *AsyncInfo); + +// Destroy the event created by __tgt_rtl_create_event previously. +int32_t __tgt_rtl_destroy_event(int32_t ID, void *Event, + __tgt_async_info *AsyncInfo); + +// Wait for the event. It can be used for setting dependences. Depending on +// targets, it can be blocking or non-blocking. +int32_t __tgt_rtl_wait_event(int32_t ID, void *Event, + __tgt_async_info *AsyncInfo); + #ifdef __cplusplus } #endif diff --git a/openmp/libomptarget/plugins/cuda/src/rtl.cpp b/openmp/libomptarget/plugins/cuda/src/rtl.cpp --- a/openmp/libomptarget/plugins/cuda/src/rtl.cpp +++ b/openmp/libomptarget/plugins/cuda/src/rtl.cpp @@ -21,6 +21,7 @@ #include #include "Debug.h" +#include "omptarget.h" #include "omptargetplugin.h" #define TARGET_NAME CUDA @@ -125,6 +126,29 @@ return OFFLOAD_SUCCESS; } +void *createEvent(__tgt_async_info *AsyncInfo) { + CUstream Stream = reinterpret_cast(AsyncInfo->Queue); + CUevent Event = nullptr; + + CUresult Err = cuEventCreate(&Event, CU_EVENT_DEFAULT); + if (Err != CUDA_SUCCESS) { + DP("Error when creating event. stream = " DPxMOD ", event = " DPxMOD "\n", + DPxPTR(Stream), DPxPTR(Event)); + CUDA_ERR_STRING(Err); + return nullptr; + } + + Err = cuEventRecord(Event, Stream); + if (Err != CUDA_SUCCESS) { + DP("Error when recording event. stream = " DPxMOD ", event = " DPxMOD "\n", + DPxPTR(Stream), DPxPTR(Event)); + CUDA_ERR_STRING(Err); + return nullptr; + } + + return (void *)Event; +} + // Structure contains per-device data struct DeviceDataTy { /// List that contains all the kernels. @@ -1103,6 +1127,39 @@ } return (Err == CUDA_SUCCESS) ? OFFLOAD_SUCCESS : OFFLOAD_FAIL; } + + int waitEvent(const int DeviceId, __tgt_async_info *AsyncInfo, + void *EventPtr) const { + CUstream Stream = reinterpret_cast(AsyncInfo->Queue); + CUevent Event = reinterpret_cast(EventPtr); + + CUresult Err = cuStreamWaitEvent(Stream, Event, CU_EVENT_WAIT_DEFAULT); + if (Err != CUDA_SUCCESS) { + DP("Error when waiting event. stream = " DPxMOD ", event = " DPxMOD "\n", + DPxPTR(Stream), DPxPTR(Event)); + CUDA_ERR_STRING(Err); + return OFFLOAD_FAIL; + } + + return OFFLOAD_SUCCESS; + } + + int destroyEvent(const int DeviceId, __tgt_async_info *AsyncInfo, + void *EventPtr) const { + CUstream Stream = reinterpret_cast(AsyncInfo->Queue); + CUevent Event = reinterpret_cast(EventPtr); + + CUresult Err = cuEventDestroy(Event); + if (Err != CUDA_SUCCESS) { + DP("Error when destroying event. stream = " DPxMOD ", event = " DPxMOD + "\n", + DPxPTR(Stream), DPxPTR(Event)); + CUDA_ERR_STRING(Err); + return OFFLOAD_FAIL; + } + + return OFFLOAD_SUCCESS; + } }; DeviceRTLTy DeviceRTL; @@ -1303,6 +1360,35 @@ InfoLevel.store(NewInfoLevel); } +void *__tgt_rtl_create_event(int32_t device_id, + __tgt_async_info *async_info_ptr) { + assert(DeviceRTL.isValidDeviceId(device_id) && "device_id is invalid"); + assert(async_info_ptr && "async_info_ptr is nullptr"); + assert(async_info_ptr->Queue && "async_info_ptr->Queue is nullptr"); + + return createEvent(async_info_ptr); +} + +int32_t __tgt_rtl_wait_event(int32_t device_id, void *event_ptr, + __tgt_async_info *async_info_ptr) { + assert(DeviceRTL.isValidDeviceId(device_id) && "device_id is invalid"); + assert(async_info_ptr && "async_info_ptr is nullptr"); + assert(async_info_ptr->Queue && "async_info_ptr->Queue is nullptr"); + assert(event_ptr && "event is nullptr"); + + return DeviceRTL.waitEvent(device_id, async_info_ptr, event_ptr); +} + +int32_t __tgt_rtl_destroy_event(int32_t device_id, void *event_ptr, + __tgt_async_info *async_info_ptr) { + assert(DeviceRTL.isValidDeviceId(device_id) && "device_id is invalid"); + assert(async_info_ptr && "async_info_ptr is nullptr"); + assert(async_info_ptr->Queue && "async_info_ptr->Queue is nullptr"); + assert(event_ptr && "event is nullptr"); + + return DeviceRTL.destroyEvent(device_id, async_info_ptr, event_ptr); +} + #ifdef __cplusplus } #endif diff --git a/openmp/libomptarget/plugins/exports b/openmp/libomptarget/plugins/exports --- a/openmp/libomptarget/plugins/exports +++ b/openmp/libomptarget/plugins/exports @@ -23,6 +23,9 @@ __tgt_rtl_unregister_lib; __tgt_rtl_supports_empty_images; __tgt_rtl_set_info_flag; + __tgt_rtl_create_event; + __tgt_rtl_destroy_event; + __tgt_rtl_wait_event; local: *; }; diff --git a/openmp/libomptarget/src/device.h b/openmp/libomptarget/src/device.h --- a/openmp/libomptarget/src/device.h +++ b/openmp/libomptarget/src/device.h @@ -13,12 +13,12 @@ #ifndef _OMPTARGET_DEVICE_H #define _OMPTARGET_DEVICE_H +#include #include #include #include #include #include -#include #include #include @@ -49,6 +49,9 @@ uintptr_t TgtPtrBegin; // target info. + /// Pointer to the event corresponding to the data update of this map. + mutable void *Event; + private: /// use mutable to allow modification via std::set iterator which is const. mutable uint64_t RefCount; @@ -65,7 +68,7 @@ HostDataToTargetTy(uintptr_t BP, uintptr_t B, uintptr_t E, uintptr_t TB, map_var_info_t Name = nullptr, bool IsINF = false) : HstPtrBase(BP), HstPtrBegin(B), HstPtrEnd(E), HstPtrName(Name), - TgtPtrBegin(TB), RefCount(IsINF ? INFRefCount : 1), + TgtPtrBegin(TB), Event(nullptr), RefCount(IsINF ? INFRefCount : 1), UpdateMtx(std::make_shared()) {} uint64_t getRefCount() const { return RefCount; } @@ -262,6 +265,12 @@ /// OFFLOAD_SUCCESS/OFFLOAD_FAIL when succeeds/fails. int32_t synchronize(AsyncInfoTy &AsyncInfo); + void *createEvent(AsyncInfoTy &AsyncInfo); + + int32_t destroyEvent(void *Event, AsyncInfoTy &AsyncInfo); + + int32_t waitEvent(void *Event, AsyncInfoTy &AsyncInfo); + private: // Call to RTL void init(); // To be called only via DeviceTy::initOnce() diff --git a/openmp/libomptarget/src/device.cpp b/openmp/libomptarget/src/device.cpp --- a/openmp/libomptarget/src/device.cpp +++ b/openmp/libomptarget/src/device.cpp @@ -195,6 +195,7 @@ AsyncInfoTy &AsyncInfo) { void *TargetPointer = nullptr; bool IsNew = false; + bool IsHostPointer = false; DataMapMtx.lock(); @@ -249,6 +250,7 @@ TargetPointer = HstPtrBegin; // We don't need to copy data in this case MoveData = false; + IsHostPointer = true; } } else if (HasPresentModifier) { DP("Mapping required by 'present' map type modifier does not exist for " @@ -276,36 +278,71 @@ // If the target pointer is valid, and we need to transfer data, issue the // data transfer. - if (TargetPointer && MoveData) { - // Lock the entry before releasing the mapping table lock such that another - // thread that could issue data movement will get the right result. - Entry->lock(); - // Release the mapping table lock right after the entry is locked. - DataMapMtx.unlock(); + if (TargetPointer) { + if (MoveData) { + // Lock the entry before releasing the mapping table lock such that + // another thread that could issue data movement will get the right + // result. + Entry->lock(); + // Release the mapping table lock right after the entry is locked. + DataMapMtx.unlock(); - DP("Moving %" PRId64 " bytes (hst:" DPxMOD ") -> (tgt:" DPxMOD ")\n", Size, - DPxPTR(HstPtrBegin), DPxPTR(TargetPointer)); + DP("Moving %" PRId64 " bytes (hst:" DPxMOD ") -> (tgt:" DPxMOD ")\n", + Size, DPxPTR(HstPtrBegin), DPxPTR(TargetPointer)); - int Ret = submitData(TargetPointer, HstPtrBegin, Size, AsyncInfo); - - // Unlock the entry immediately after the data movement is issued. - Entry->unlock(); - - if (Ret != OFFLOAD_SUCCESS) { - REPORT("Copying data to device failed.\n"); + // Issue data movement. + int Ret = submitData(TargetPointer, HstPtrBegin, Size, AsyncInfo); // We will also return nullptr if the data movement fails because that // pointer points to a corrupted memory region so it doesn't make any // sense to continue to use it. - return {{false /* IsNewEntry */, false /* IsHostPointer */}, - {} /* Entry */, - nullptr /* TargetPointer */}; + if (Ret != OFFLOAD_SUCCESS) { + Entry->unlock(); + + REPORT("Copying data to device failed.\n"); + return {{false /* IsNewEntry */, false /* IsHostPointer */}, + {} /* MapTableEntry */, + nullptr /* TargetPointer */}; + } + // Create an event at this moment. + void *Event = createEvent(AsyncInfo); + void *OldEvent = Entry->Event; + // Set the new event. + Entry->Event = Event; + // We're done with the entry. Release the entry. + Entry->unlock(); + // If there is an event attached, destroy it. + if (OldEvent) { + // TODO: Do we want to check the return value here? + destroyEvent(OldEvent, AsyncInfo); + } + } else { + // Release the mapping table lock directly. + DataMapMtx.unlock(); + // If not a host pointer, we need to wait for the event if it exists. + if (!IsHostPointer) { + Entry->lock(); + void *Event = Entry->Event; + Entry->unlock(); + + if (Event) { + int Ret = waitEvent(Event, AsyncInfo); + if (Ret != OFFLOAD_SUCCESS) { + // If it fails to wait for the event, we need to return nullptr in + // case of any data race. + REPORT("Failed to wait for event " DPxMOD ".\n", DPxPTR(Event)); + return {{false /* IsNewEntry */, false /* IsHostPointer */}, + {} /* MapTableEntry */, + nullptr /* TargetPointer */}; + } + } + } } } else { // Release the mapping table lock directly. DataMapMtx.unlock(); } - return {{IsNew, false /* IsHostPointer */}, Entry, TargetPointer}; + return {{IsNew, IsHostPointer}, Entry, TargetPointer}; } // Used by targetDataBegin, targetDataEnd, targetDataUpdate and target. @@ -547,6 +584,27 @@ return OFFLOAD_SUCCESS; } +void *DeviceTy::createEvent(AsyncInfoTy &AsyncInfo) { + if (RTL->create_event) + return RTL->create_event(RTLDeviceID, AsyncInfo); + + return nullptr; +} + +int32_t DeviceTy::destroyEvent(void *Event, AsyncInfoTy &AsyncInfo) { + if (RTL->create_event) + return RTL->destroy_event(RTLDeviceID, Event, AsyncInfo); + + return OFFLOAD_SUCCESS; +} + +int32_t DeviceTy::waitEvent(void *Event, AsyncInfoTy &AsyncInfo) { + if (RTL->create_event) + return RTL->wait_event(RTLDeviceID, Event, AsyncInfo); + + return OFFLOAD_SUCCESS; +} + /// Check whether a device has an associated RTL and initialize it if it's not /// already initialized. bool device_is_ready(int device_num) { diff --git a/openmp/libomptarget/src/omptarget.cpp b/openmp/libomptarget/src/omptarget.cpp --- a/openmp/libomptarget/src/omptarget.cpp +++ b/openmp/libomptarget/src/omptarget.cpp @@ -487,26 +487,21 @@ // entry for a global that might not already be allocated by the time the // PTR_AND_OBJ entry is handled below, and so the allocation might fail // when HasPresentModifier. - Pointer_TPR = Device.getOrAllocTgtPtr( + Pointer_TPR = Device.getTargetPointer( HstPtrBase, HstPtrBase, sizeof(void *), nullptr, false, IsImplicit, - UpdateRef, HasCloseModifier, HasPresentModifier); + UpdateRef, HasCloseModifier, HasPresentModifier, AsyncInfo); PointerTgtPtrBegin = Pointer_TPR.TargetPointer; IsHostPtr = Pointer_TPR.Flags.IsHostPointer; if (!PointerTgtPtrBegin) { - REPORT("Call to getOrAllocTgtPtr returned null pointer (%s).\n", + REPORT("Call to getTargetPointer returned null pointer (%s).\n", HasPresentModifier ? "'present' map type modifier" : "device failure or illegal mapping"); return OFFLOAD_FAIL; } -<<<<<<< HEAD - DP("There are %zu bytes allocated at target address " DPxMOD "\n", - sizeof(void *), DPxPTR(PointerTgtPtrBegin)); -======= DP("There are %zu bytes allocated at target address " DPxMOD " - is%s new" "\n", sizeof(void *), DPxPTR(PointerTgtPtrBegin), (Pointer_TPR.Flags.IsNewEntry ? "" : " not")); ->>>>>>> a18770033db8... [OpenMP][Offloading] Refined return value of `DeviceTy::getOrAllocTgtPtr` Pointer_HstPtrBegin = HstPtrBase; // modify current entry. HstPtrBase = *(void **)HstPtrBase; @@ -566,13 +561,28 @@ uint64_t Delta = (uint64_t)HstPtrBegin - (uint64_t)HstPtrBase; void *&TgtPtrBase = AsyncInfo.getVoidPtrLocation(); TgtPtrBase = (void *)((uint64_t)TgtPtrBegin - Delta); - int rt = Device.submitData(PointerTgtPtrBegin, &TgtPtrBase, - sizeof(void *), AsyncInfo); - if (rt != OFFLOAD_SUCCESS) { + + Pointer_TPR.MapTableEntry->lock(); + int Ret = Device.submitData(PointerTgtPtrBegin, &TgtPtrBase, + sizeof(void *), AsyncInfo); + if (Ret != OFFLOAD_SUCCESS) { + Pointer_TPR.MapTableEntry->unlock(); + REPORT("Copying data to device failed.\n"); return OFFLOAD_FAIL; } - // TODO: Attach the event in AsyncInfo to the map table entry if needed + // Create a new event for this moment + void *Event = Device.createEvent(AsyncInfo); + // Exchange the old event with new created event + void *OldEvent = Pointer_TPR.MapTableEntry->Event; + Pointer_TPR.MapTableEntry->Event = Event; + Pointer_TPR.MapTableEntry->unlock(); + + // If the old event is not null, we need to destroy it. + if (OldEvent) { + // TODO: Do we need to deal with the failure? + Device.destroyEvent(OldEvent, AsyncInfo); + } // create shadow pointers for this entry Device.ShadowMtx.lock(); Device.ShadowPtrMap[Pointer_HstPtrBegin] = { diff --git a/openmp/libomptarget/src/rtl.h b/openmp/libomptarget/src/rtl.h --- a/openmp/libomptarget/src/rtl.h +++ b/openmp/libomptarget/src/rtl.h @@ -56,6 +56,9 @@ typedef int32_t (*register_lib_ty)(__tgt_bin_desc *); typedef int32_t(supports_empty_images_ty)(); typedef void(set_info_flag_ty)(uint32_t); + typedef void *(create_event_ty)(int32_t, __tgt_async_info *); + typedef int32_t(destroy_event_ty)(int32_t, void *, __tgt_async_info *); + typedef int32_t(wait_event_ty)(int32_t, void *, __tgt_async_info *); int32_t Idx = -1; // RTL index, index is the number of devices // of other RTLs that were registered before, @@ -93,6 +96,9 @@ register_lib_ty unregister_lib = nullptr; supports_empty_images_ty *supports_empty_images = nullptr; set_info_flag_ty *set_info_flag = nullptr; + create_event_ty *create_event = nullptr; + destroy_event_ty *destroy_event = nullptr; + wait_event_ty *wait_event = nullptr; // Are there images associated with this RTL. bool isUsed = false; diff --git a/openmp/libomptarget/src/rtl.cpp b/openmp/libomptarget/src/rtl.cpp --- a/openmp/libomptarget/src/rtl.cpp +++ b/openmp/libomptarget/src/rtl.cpp @@ -177,6 +177,11 @@ dlsym(dynlib_handle, "__tgt_rtl_supports_empty_images"); *((void **)&R.set_info_flag) = dlsym(dynlib_handle, "__tgt_rtl_set_info_flag"); + *((void **)&R.create_event) = + dlsym(dynlib_handle, "__tgt_rtl_create_event"); + *((void **)&R.destroy_event) = + dlsym(dynlib_handle, "__tgt_rtl_destroy_event"); + *((void **)&R.wait_event) = dlsym(dynlib_handle, "__tgt_rtl_wait_event"); } DP("RTLs loaded!\n");