diff --git a/openmp/libomptarget/include/device.h b/openmp/libomptarget/include/device.h --- a/openmp/libomptarget/include/device.h +++ b/openmp/libomptarget/include/device.h @@ -435,6 +435,12 @@ /// OFFLOAD_SUCCESS/OFFLOAD_FAIL when succeeds/fails. int32_t synchronize(AsyncInfoTy &AsyncInfo); + /// Query for device/queue/event based completion on \p AsyncInfo in a + /// non-blocking manner and return OFFLOAD_SUCCESS/OFFLOAD_FAIL when + /// succeeds/fails. Must be called multiple times until AsyncInfo is + /// completed and AsyncInfo.isDone() returns true. + int32_t queryAsync(AsyncInfoTy &AsyncInfo); + /// Calls the corresponding print in the \p RTLDEVID /// device RTL to obtain the information of the specific device. bool printDeviceInfo(int32_t RTLDevID); diff --git a/openmp/libomptarget/include/omptarget.h b/openmp/libomptarget/include/omptarget.h --- a/openmp/libomptarget/include/omptarget.h +++ b/openmp/libomptarget/include/omptarget.h @@ -15,11 +15,15 @@ #define _OMPTARGET_H_ #include +#include #include #include +#include #include +#include "llvm/ADT/SmallVector.h" + #define OFFLOAD_SUCCESS (0) #define OFFLOAD_FAIL (~0) @@ -185,10 +189,21 @@ /// as long as this AsyncInfoTy object. std::deque BufferLocations; + /// Post-processing operations executed after a successful synchronization. + /// \note the post-processing function should return OFFLOAD_SUCCESS or + /// OFFLOAD_FAIL appropriately. + using PostProcFuncTy = std::function; + llvm::SmallVector PostProcessingFunctions; + __tgt_async_info AsyncInfo; DeviceTy &Device; public: + enum class SyncType { + BLOCKING, + NON_BLOCKING + }; + AsyncInfoTy(DeviceTy &Device) : Device(Device) {} ~AsyncInfoTy() { synchronize(); } @@ -199,11 +214,30 @@ /// Synchronize all pending actions. /// /// \returns OFFLOAD_FAIL or OFFLOAD_SUCCESS appropriately. - int synchronize(); + int synchronize(SyncType SyncType = SyncType::BLOCKING); /// Return a void* reference with a lifetime that is at least as long as this /// AsyncInfoTy object. The location can be used as intermediate buffer. void *&getVoidPtrLocation(); + + /// Check if all asynchronous operations are completed. + /// + /// \returns true if there is no pending asynchronous operations, false + /// otherwise. + bool isDone(); + + /// Add a new post-processing function to be executed after synchronization. + /// + /// \param[in] Function is a templated function (e.g., function pointers, + /// lambdas, std::function) that can be convertible to a PostProcFuncTy (i.e., + /// it must have int() as its function signature). + template + void addPostProcessingFunction(FuncTy &&Function) { + static_assert(std::is_convertible_v, + "Invalid post-processing function type. Please check " + "function signature!"); + PostProcessingFunctions.emplace_back(Function); + } }; /// This struct is a record of non-contiguous information diff --git a/openmp/libomptarget/include/omptargetplugin.h b/openmp/libomptarget/include/omptargetplugin.h --- a/openmp/libomptarget/include/omptargetplugin.h +++ b/openmp/libomptarget/include/omptargetplugin.h @@ -155,6 +155,16 @@ // error code. int32_t __tgt_rtl_synchronize(int32_t ID, __tgt_async_info *AsyncInfo); +// Queries for the completion of asynchronous operations. Instead of blocking +// the calling thread as __tgt_rtl_synchronize, the progress of the operations +// stored in AsyncInfo->Queue is queried in a non-blocking manner, partially +// advancing their execution. If all operations are completed, AsyncInfo->Queue +// is set to nullptr. If there are still pending operations, AsyncInfo->Queue is +// kept as a valid queue. In any case of success (i.e., successful query +// with/without completing all operations), return zero. Otherwise, return an +// error code. +int32_t __tgt_rtl_query_async(int32_t ID, __tgt_async_info *AsyncInfo); + // Set plugin's internal information flag externally. void __tgt_rtl_set_info_flag(uint32_t); diff --git a/openmp/libomptarget/include/rtl.h b/openmp/libomptarget/include/rtl.h --- a/openmp/libomptarget/include/rtl.h +++ b/openmp/libomptarget/include/rtl.h @@ -58,6 +58,7 @@ __tgt_async_info *); typedef int64_t(init_requires_ty)(int64_t); typedef int32_t(synchronize_ty)(int32_t, __tgt_async_info *); + typedef int32_t(query_async_ty)(int32_t, __tgt_async_info *); typedef int32_t (*register_lib_ty)(__tgt_bin_desc *); typedef int32_t(supports_empty_images_ty)(); typedef void(print_device_info_ty)(int32_t); @@ -108,6 +109,7 @@ run_team_region_async_ty *run_team_region_async = nullptr; init_requires_ty *init_requires = nullptr; synchronize_ty *synchronize = nullptr; + query_async_ty *query_async = nullptr; register_lib_ty register_lib = nullptr; register_lib_ty unregister_lib = nullptr; supports_empty_images_ty *supports_empty_images = nullptr; diff --git a/openmp/libomptarget/plugins/cuda/dynamic_cuda/cuda.h b/openmp/libomptarget/plugins/cuda/dynamic_cuda/cuda.h --- a/openmp/libomptarget/plugins/cuda/dynamic_cuda/cuda.h +++ b/openmp/libomptarget/plugins/cuda/dynamic_cuda/cuda.h @@ -29,6 +29,7 @@ CUDA_ERROR_INVALID_VALUE = 1, CUDA_ERROR_NO_DEVICE = 100, CUDA_ERROR_INVALID_HANDLE = 400, + CUDA_ERROR_NOT_READY = 600, } CUresult; typedef enum CUstream_flags_enum { @@ -241,6 +242,7 @@ CUresult cuStreamCreate(CUstream *, unsigned); CUresult cuStreamDestroy(CUstream); CUresult cuStreamSynchronize(CUstream); +CUresult cuStreamQuery(CUstream); CUresult cuCtxSetCurrent(CUcontext); CUresult cuDevicePrimaryCtxRelease(CUdevice); CUresult cuDevicePrimaryCtxGetState(CUdevice, unsigned *, int *); diff --git a/openmp/libomptarget/plugins/cuda/dynamic_cuda/cuda.cpp b/openmp/libomptarget/plugins/cuda/dynamic_cuda/cuda.cpp --- a/openmp/libomptarget/plugins/cuda/dynamic_cuda/cuda.cpp +++ b/openmp/libomptarget/plugins/cuda/dynamic_cuda/cuda.cpp @@ -57,6 +57,7 @@ DLWRAP(cuStreamCreate, 2); DLWRAP(cuStreamDestroy, 1); DLWRAP(cuStreamSynchronize, 1); +DLWRAP(cuStreamQuery, 1); DLWRAP(cuCtxSetCurrent, 1); DLWRAP(cuDevicePrimaryCtxRelease, 1); DLWRAP(cuDevicePrimaryCtxGetState, 3); diff --git a/openmp/libomptarget/plugins/cuda/src/rtl.cpp b/openmp/libomptarget/plugins/cuda/src/rtl.cpp --- a/openmp/libomptarget/plugins/cuda/src/rtl.cpp +++ b/openmp/libomptarget/plugins/cuda/src/rtl.cpp @@ -1263,6 +1263,31 @@ return (Err == CUDA_SUCCESS) ? OFFLOAD_SUCCESS : OFFLOAD_FAIL; } + int queryAsync(const int DeviceId, __tgt_async_info *AsyncInfo) const { + CUstream Stream = reinterpret_cast(AsyncInfo->Queue); + CUresult Err = cuStreamQuery(Stream); + + if (Err == CUDA_ERROR_NOT_READY) { + // Not ready streams must be considered as successful operations. + Err = CUDA_SUCCESS; + } else { + // Once the stream is synchronized or an error occurs, return it to the + // stream pool and reset AsyncInfo. This is to make sure the + // synchronization only works for its own tasks. + StreamPool[DeviceId]->release( + reinterpret_cast(AsyncInfo->Queue)); + AsyncInfo->Queue = nullptr; + } + + if (Err != CUDA_SUCCESS) { + DP("Error when querying for stream progress. stream = " DPxMOD + ", async info ptr = " DPxMOD "\n", + DPxPTR(Stream), DPxPTR(AsyncInfo)); + CUDA_ERR_STRING(Err); + } + return (Err == CUDA_SUCCESS) ? OFFLOAD_SUCCESS : OFFLOAD_FAIL; + } + void printDeviceInfo(int32_t DeviceId) { char TmpChar[1000]; std::string TmpStr; @@ -1776,6 +1801,15 @@ return DeviceRTL.synchronize(DeviceId, AsyncInfoPtr); } +int32_t __tgt_rtl_query_async(int32_t DeviceId, + __tgt_async_info *AsyncInfoPtr) { + assert(DeviceRTL.isValidDeviceId(DeviceId) && "device_id is invalid"); + assert(AsyncInfoPtr && "async_info_ptr is nullptr"); + assert(AsyncInfoPtr->Queue && "async_info_ptr->Queue is nullptr"); + // NOTE: We don't need to set context for stream query. + return DeviceRTL.queryAsync(DeviceId, AsyncInfoPtr); +} + void __tgt_rtl_set_info_flag(uint32_t NewInfoLevel) { std::atomic &InfoLevel = getInfoLevelInternal(); InfoLevel.store(NewInfoLevel); diff --git a/openmp/libomptarget/plugins/exports b/openmp/libomptarget/plugins/exports --- a/openmp/libomptarget/plugins/exports +++ b/openmp/libomptarget/plugins/exports @@ -23,6 +23,7 @@ __tgt_rtl_run_target_region; __tgt_rtl_run_target_region_async; __tgt_rtl_synchronize; + __tgt_rtl_query_async; __tgt_rtl_register_lib; __tgt_rtl_unregister_lib; __tgt_rtl_supports_empty_images; diff --git a/openmp/libomptarget/src/device.cpp b/openmp/libomptarget/src/device.cpp --- a/openmp/libomptarget/src/device.cpp +++ b/openmp/libomptarget/src/device.cpp @@ -628,6 +628,13 @@ return OFFLOAD_SUCCESS; } +int32_t DeviceTy::queryAsync(AsyncInfoTy &AsyncInfo) { + if (RTL->query_async) + return RTL->query_async(RTLDeviceID, AsyncInfo); + + return synchronize(AsyncInfo); +} + int32_t DeviceTy::createEvent(void **Event) { if (RTL->create_event) return RTL->create_event(RTLDeviceID, Event); diff --git a/openmp/libomptarget/src/interface.cpp b/openmp/libomptarget/src/interface.cpp --- a/openmp/libomptarget/src/interface.cpp +++ b/openmp/libomptarget/src/interface.cpp @@ -61,6 +61,39 @@ } } +static inline void targetDataMapper( + ident_t *Loc, DeviceTy &Device, int64_t DeviceId, int32_t ArgNum, + void **ArgsBase, void **Args, int64_t *ArgSizes, int64_t *ArgTypes, + map_var_info_t *ArgNames, void **ArgMappers, AsyncInfoTy &AsyncInfo, + TargetDataFuncPtrTy TargetDataFunction, const char *RegionTypeMsg, + AsyncInfoTy::SyncType SyncType = AsyncInfoTy::SyncType::BLOCKING, + bool Dispatch = true, bool FromMapper = false) { + TIMESCOPE_WITH_IDENT(Loc); + + if (getInfoLevel() & OMP_INFOTYPE_KERNEL_ARGS) + printKernelArguments(Loc, DeviceId, ArgNum, ArgSizes, ArgTypes, ArgNames, + RegionTypeMsg); +#ifdef OMPTARGET_DEBUG + for (int I = 0; I < ArgNum; ++I) { + DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64 + ", Type=0x%" PRIx64 ", Name=%s\n", + I, DPxPTR(ArgsBase[I]), DPxPTR(Args[I]), ArgSizes[I], ArgTypes[I], + (ArgNames) ? getNameFromMapping(ArgNames[I]).c_str() : "unknown"); + } +#endif + + int Rc = OFFLOAD_SUCCESS; + if (Dispatch) + Rc = TargetDataFunction(Loc, Device, ArgNum, ArgsBase, Args, ArgSizes, + ArgTypes, ArgNames, ArgMappers, AsyncInfo, + FromMapper); + + if (Rc == OFFLOAD_SUCCESS) + Rc = AsyncInfo.synchronize(SyncType); + + handleTargetOutcome(Rc == OFFLOAD_SUCCESS, Loc); +} + /// creates host-to-target data mapping, stores it in the /// libomptarget.so internal structure (an entry in a stack of data maps) /// and passes the data to the device. @@ -71,8 +104,10 @@ map_var_info_t *ArgNames, void **ArgMappers) { TIMESCOPE_WITH_IDENT(Loc); + DP("Entering data begin region for device %" PRId64 " with %d mappings\n", DeviceId, ArgNum); + if (checkDeviceAndCtors(DeviceId, Loc)) { DP("Not offloading to device %" PRId64 "\n", DeviceId); return; @@ -80,24 +115,10 @@ DeviceTy &Device = *PM->Devices[DeviceId]; - if (getInfoLevel() & OMP_INFOTYPE_KERNEL_ARGS) - printKernelArguments(Loc, DeviceId, ArgNum, ArgSizes, ArgTypes, ArgNames, - "Entering OpenMP data region"); -#ifdef OMPTARGET_DEBUG - for (int I = 0; I < ArgNum; ++I) { - DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64 - ", Type=0x%" PRIx64 ", Name=%s\n", - I, DPxPTR(ArgsBase[I]), DPxPTR(Args[I]), ArgSizes[I], ArgTypes[I], - (ArgNames) ? getNameFromMapping(ArgNames[I]).c_str() : "unknown"); - } -#endif - AsyncInfoTy AsyncInfo(Device); - int Rc = targetDataBegin(Loc, Device, ArgNum, ArgsBase, Args, ArgSizes, - ArgTypes, ArgNames, ArgMappers, AsyncInfo); - if (Rc == OFFLOAD_SUCCESS) - Rc = AsyncInfo.synchronize(); - handleTargetOutcome(Rc == OFFLOAD_SUCCESS, Loc); + targetDataMapper(Loc, Device, DeviceId, ArgNum, ArgsBase, Args, ArgSizes, + ArgTypes, ArgNames, ArgMappers, AsyncInfo, targetDataBegin, + "Entering OpenMP data region"); } EXTERN void __tgt_target_data_begin_nowait_mapper( @@ -107,8 +128,21 @@ void *NoAliasDepList) { TIMESCOPE_WITH_IDENT(Loc); - __tgt_target_data_begin_mapper(Loc, DeviceId, ArgNum, ArgsBase, Args, - ArgSizes, ArgTypes, ArgNames, ArgMappers); + DP("Entering data begin region for device %" PRId64 " with %d mappings\n", + DeviceId, ArgNum); + + if (checkDeviceAndCtors(DeviceId, Loc)) { + DP("Not offloading to device %" PRId64 "\n", DeviceId); + return; + } + + DeviceTy &Device = *PM->Devices[DeviceId]; + + TaskAsyncInfoTy AsyncInfo(Device); + targetDataMapper(Loc, Device, DeviceId, ArgNum, ArgsBase, Args, ArgSizes, + ArgTypes, ArgNames, ArgMappers, *AsyncInfo, targetDataBegin, + "Entering OpenMP data region", AsyncInfo.getSyncType(), + AsyncInfo.shouldDispatch()); } /// passes data from the target, releases target memory and destroys @@ -121,7 +155,10 @@ map_var_info_t *ArgNames, void **ArgMappers) { TIMESCOPE_WITH_IDENT(Loc); - DP("Entering data end region with %d mappings\n", ArgNum); + + DP("Entering data end region for device %" PRId64 " with %d mappings\n", + DeviceId, ArgNum); + if (checkDeviceAndCtors(DeviceId, Loc)) { DP("Not offloading to device %" PRId64 "\n", DeviceId); return; @@ -129,24 +166,10 @@ DeviceTy &Device = *PM->Devices[DeviceId]; - if (getInfoLevel() & OMP_INFOTYPE_KERNEL_ARGS) - printKernelArguments(Loc, DeviceId, ArgNum, ArgSizes, ArgTypes, ArgNames, - "Exiting OpenMP data region"); -#ifdef OMPTARGET_DEBUG - for (int I = 0; I < ArgNum; ++I) { - DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64 - ", Type=0x%" PRIx64 ", Name=%s\n", - I, DPxPTR(ArgsBase[I]), DPxPTR(Args[I]), ArgSizes[I], ArgTypes[I], - (ArgNames) ? getNameFromMapping(ArgNames[I]).c_str() : "unknown"); - } -#endif - AsyncInfoTy AsyncInfo(Device); - int Rc = targetDataEnd(Loc, Device, ArgNum, ArgsBase, Args, ArgSizes, - ArgTypes, ArgNames, ArgMappers, AsyncInfo); - if (Rc == OFFLOAD_SUCCESS) - Rc = AsyncInfo.synchronize(); - handleTargetOutcome(Rc == OFFLOAD_SUCCESS, Loc); + targetDataMapper(Loc, Device, DeviceId, ArgNum, ArgsBase, Args, ArgSizes, + ArgTypes, ArgNames, ArgMappers, AsyncInfo, targetDataEnd, + "Exiting OpenMP data region"); } EXTERN void __tgt_target_data_end_nowait_mapper( @@ -156,8 +179,21 @@ void *NoAliasDepList) { TIMESCOPE_WITH_IDENT(Loc); - __tgt_target_data_end_mapper(Loc, DeviceId, ArgNum, ArgsBase, Args, ArgSizes, - ArgTypes, ArgNames, ArgMappers); + DP("Entering data end region for device %" PRId64 " with %d mappings\n", + DeviceId, ArgNum); + + if (checkDeviceAndCtors(DeviceId, Loc)) { + DP("Not offloading to device %" PRId64 "\n", DeviceId); + return; + } + + DeviceTy &Device = *PM->Devices[DeviceId]; + + TaskAsyncInfoTy AsyncInfo(Device); + targetDataMapper(Loc, Device, DeviceId, ArgNum, ArgsBase, Args, ArgSizes, + ArgTypes, ArgNames, ArgMappers, *AsyncInfo, targetDataEnd, + "Exiting OpenMP data region", AsyncInfo.getSyncType(), + AsyncInfo.shouldDispatch()); } EXTERN void __tgt_target_data_update_mapper(ident_t *Loc, int64_t DeviceId, @@ -167,23 +203,21 @@ map_var_info_t *ArgNames, void **ArgMappers) { TIMESCOPE_WITH_IDENT(Loc); - DP("Entering data update with %d mappings\n", ArgNum); + + DP("Entering data update region for device %" PRId64 " with %d mappings\n", + DeviceId, ArgNum); + if (checkDeviceAndCtors(DeviceId, Loc)) { DP("Not offloading to device %" PRId64 "\n", DeviceId); return; } - if (getInfoLevel() & OMP_INFOTYPE_KERNEL_ARGS) - printKernelArguments(Loc, DeviceId, ArgNum, ArgSizes, ArgTypes, ArgNames, - "Updating OpenMP data"); - DeviceTy &Device = *PM->Devices[DeviceId]; + AsyncInfoTy AsyncInfo(Device); - int Rc = targetDataUpdate(Loc, Device, ArgNum, ArgsBase, Args, ArgSizes, - ArgTypes, ArgNames, ArgMappers, AsyncInfo); - if (Rc == OFFLOAD_SUCCESS) - Rc = AsyncInfo.synchronize(); - handleTargetOutcome(Rc == OFFLOAD_SUCCESS, Loc); + targetDataMapper(Loc, Device, DeviceId, ArgNum, ArgsBase, Args, ArgSizes, + ArgTypes, ArgNames, ArgMappers, AsyncInfo, targetDataUpdate, + "Updating OpenMP data"); } EXTERN void __tgt_target_data_update_nowait_mapper( @@ -193,35 +227,34 @@ void *NoAliasDepList) { TIMESCOPE_WITH_IDENT(Loc); - __tgt_target_data_update_mapper(Loc, DeviceId, ArgNum, ArgsBase, Args, - ArgSizes, ArgTypes, ArgNames, ArgMappers); + DP("Entering data update region for device %" PRId64 " with %d mappings\n", + DeviceId, ArgNum); + + if (checkDeviceAndCtors(DeviceId, Loc)) { + DP("Not offloading to device %" PRId64 "\n", DeviceId); + return; + } + + DeviceTy &Device = *PM->Devices[DeviceId]; + + TaskAsyncInfoTy AsyncInfo(Device); + targetDataMapper(Loc, Device, DeviceId, ArgNum, ArgsBase, Args, ArgSizes, + ArgTypes, ArgNames, ArgMappers, *AsyncInfo, targetDataUpdate, + "Updating OpenMP data", AsyncInfo.getSyncType(), + AsyncInfo.shouldDispatch()); } -/// Implements a kernel entry that executes the target region on the specified -/// device. -/// -/// \param Loc Source location associated with this target region. -/// \param DeviceId The device to execute this region, -1 indicated the default. -/// \param NumTeams Number of teams to launch the region with, -1 indicates a -/// non-teams region and 0 indicates it was unspecified. -/// \param ThreadLimit Limit to the number of threads to use in the kernel -/// launch, 0 indicates it was unspecified. -/// \param HostPtr The pointer to the host function registered with the kernel. -/// \param Args All arguments to this kernel launch (see struct definition). -EXTERN int __tgt_target_kernel(ident_t *Loc, int64_t DeviceId, int32_t NumTeams, - int32_t ThreadLimit, void *HostPtr, - __tgt_kernel_arguments *Args) { +static inline int +targetKernel(ident_t *Loc, DeviceTy &Device, int64_t DeviceId, int32_t NumTeams, + int32_t ThreadLimit, void *HostPtr, __tgt_kernel_arguments *Args, + AsyncInfoTy &AsyncInfo, + AsyncInfoTy::SyncType SyncType = AsyncInfoTy::SyncType::BLOCKING, + bool Dispatch = true) { TIMESCOPE_WITH_IDENT(Loc); - DP("Entering target region with entry point " DPxMOD " and device Id %" PRId64 - "\n", - DPxPTR(HostPtr), DeviceId); + if (Args->Version != 1) { DP("Unexpected ABI version: %d\n", Args->Version); } - if (checkDeviceAndCtors(DeviceId, Loc)) { - DP("Not offloading to device %" PRId64 "\n", DeviceId); - return OMP_TGT_FAIL; - } if (getInfoLevel() & OMP_INFOTYPE_KERNEL_ARGS) printKernelArguments(Loc, DeviceId, Args->NumArgs, Args->ArgSizes, @@ -242,27 +275,75 @@ if (!IsTeams) NumTeams = 0; - DeviceTy &Device = *PM->Devices[DeviceId]; - AsyncInfoTy AsyncInfo(Device); - int Rc = target(Loc, Device, HostPtr, Args->NumArgs, Args->ArgBasePtrs, - Args->ArgPtrs, Args->ArgSizes, Args->ArgTypes, Args->ArgNames, - Args->ArgMappers, NumTeams, ThreadLimit, Args->Tripcount, - IsTeams, AsyncInfo); + int Rc = OFFLOAD_SUCCESS; + if (AsyncInfo.isDone()) + Rc = target(Loc, Device, HostPtr, Args->NumArgs, Args->ArgBasePtrs, + Args->ArgPtrs, Args->ArgSizes, Args->ArgTypes, Args->ArgNames, + Args->ArgMappers, NumTeams, ThreadLimit, Args->Tripcount, + IsTeams, AsyncInfo); + if (Rc == OFFLOAD_SUCCESS) - Rc = AsyncInfo.synchronize(); + Rc = AsyncInfo.synchronize(SyncType); + handleTargetOutcome(Rc == OFFLOAD_SUCCESS, Loc); assert(Rc == OFFLOAD_SUCCESS && "__tgt_target_kernel unexpected failure!"); + return OMP_TGT_SUCCESS; } +/// Implements a kernel entry that executes the target region on the specified +/// device. +/// +/// \param Loc Source location associated with this target region. +/// \param DeviceId The device to execute this region, -1 indicated the default. +/// \param NumTeams Number of teams to launch the region with, -1 indicates a +/// non-teams region and 0 indicates it was unspecified. +/// \param ThreadLimit Limit to the number of threads to use in the kernel +/// launch, 0 indicates it was unspecified. +/// \param HostPtr The pointer to the host function registered with the kernel. +/// \param Args All arguments to this kernel launch (see struct definition). +EXTERN int __tgt_target_kernel(ident_t *Loc, int64_t DeviceId, int32_t NumTeams, + int32_t ThreadLimit, void *HostPtr, + __tgt_kernel_arguments *Args) { + TIMESCOPE_WITH_IDENT(Loc); + + DP("Entering target region with entry point " DPxMOD " and device Id %" PRId64 + "\n", + DPxPTR(HostPtr), DeviceId); + + if (checkDeviceAndCtors(DeviceId, Loc)) { + DP("Not offloading to device %" PRId64 "\n", DeviceId); + return OMP_TGT_FAIL; + } + + DeviceTy &Device = *PM->Devices[DeviceId]; + + AsyncInfoTy AsyncInfo(Device); + return targetKernel(Loc, Device, DeviceId, NumTeams, ThreadLimit, HostPtr, + Args, AsyncInfo); +} + EXTERN int __tgt_target_kernel_nowait( ident_t *Loc, int64_t DeviceId, int32_t NumTeams, int32_t ThreadLimit, void *HostPtr, __tgt_kernel_arguments *Args, int32_t DepNum, void *DepList, int32_t NoAliasDepNum, void *NoAliasDepList) { TIMESCOPE_WITH_IDENT(Loc); - return __tgt_target_kernel(Loc, DeviceId, NumTeams, ThreadLimit, HostPtr, - Args); + DP("Entering target region with entry point " DPxMOD " and device Id %" PRId64 + "\n", + DPxPTR(HostPtr), DeviceId); + + if (checkDeviceAndCtors(DeviceId, Loc)) { + DP("Not offloading to device %" PRId64 "\n", DeviceId); + return OMP_TGT_FAIL; + } + + DeviceTy &Device = *PM->Devices[DeviceId]; + + TaskAsyncInfoTy AsyncInfo(Device); + return targetKernel(Loc, Device, DeviceId, NumTeams, ThreadLimit, HostPtr, + Args, *AsyncInfo, AsyncInfo.getSyncType(), + AsyncInfo.shouldDispatch()); } // Get the current number of components for a user-defined mapper. diff --git a/openmp/libomptarget/src/omptarget.cpp b/openmp/libomptarget/src/omptarget.cpp --- a/openmp/libomptarget/src/omptarget.cpp +++ b/openmp/libomptarget/src/omptarget.cpp @@ -22,15 +22,33 @@ using llvm::SmallVector; -int AsyncInfoTy::synchronize() { +int AsyncInfoTy::synchronize(SyncType SyncType) { int Result = OFFLOAD_SUCCESS; if (AsyncInfo.Queue) { - // If we have a queue we need to synchronize it now. - Result = Device.synchronize(*this); - assert(AsyncInfo.Queue == nullptr && - "The device plugin should have nulled the queue to indicate there " - "are no outstanding actions!"); + switch (SyncType) { + case SyncType::BLOCKING: + // If we have a queue we need to synchronize it now. + Result = Device.synchronize(*this); + assert(AsyncInfo.Queue == nullptr && + "The device plugin should have nulled the queue to indicate there " + "are no outstanding actions!"); + break; + case SyncType::NON_BLOCKING: + Result = Device.queryAsync(*this); + break; + } + } + + // Run any pending post-processing function registered on this async object. + if (Result == OFFLOAD_SUCCESS && isDone()) { + for (auto &PostProcFunc : PostProcessingFunctions) { + Result = PostProcFunc(); + if (Result != OFFLOAD_SUCCESS) + break; + } + PostProcessingFunctions.clear(); } + return Result; } @@ -39,6 +57,8 @@ return BufferLocations.back(); } +bool AsyncInfoTy::isDone() { return AsyncInfo.Queue == nullptr; } + /* All begin addresses for partially mapped structs must be 8-aligned in order * to ensure proper alignment of members. E.g. * @@ -675,12 +695,89 @@ } // namespace +/// Applies the necessary post-processing procedures to entries listed in \p +/// EntriesInfo after the execution of all device side operations from a target +/// data end. This includes the update of pointers at the host and removal of +/// device buffer when needed. It returns OFFLOAD_FAIL or OFFLOAD_SUCCESS +/// according to the successfulness of the operations. +static int +postProcessingTargetDataEnd(DeviceTy *Device, + SmallVector EntriesInfo, + void * FromMapperBase) { + int Ret = OFFLOAD_SUCCESS; + + for (PostProcessingInfo &Info : EntriesInfo) { + // If we marked the entry to be deleted we need to verify no other + // thread reused it by now. If deletion is still supposed to happen by + // this thread LR will be set and exclusive access to the HDTT map + // will avoid another thread reusing the entry now. Note that we do + // not request (exclusive) access to the HDTT map if Info.DelEntry is + // not set. + LookupResult LR; + DeviceTy::HDTTMapAccessorTy HDTTMap = + Device->HostDataToTargetMap.getExclusiveAccessor(!Info.DelEntry); + + if (Info.DelEntry) { + LR = Device->lookupMapping(HDTTMap, Info.HstPtrBegin, Info.DataSize); + if (LR.Entry->getTotalRefCount() != 0 || + LR.Entry->getDeleteThreadId() != std::this_thread::get_id()) { + // The thread is not in charge of deletion anymore. Give up access + // to the HDTT map and unset the deletion flag. + HDTTMap.destroy(); + Info.DelEntry = false; + } + } + + // If we copied back to the host a struct/array containing pointers, + // we need to restore the original host pointer values from their + // shadow copies. If the struct is going to be deallocated, remove any + // remaining shadow pointer entries for this struct. + auto CB = [&](ShadowPtrListTy::iterator &Itr) { + // If we copied the struct to the host, we need to restore the + // pointer. + if (Info.ArgType & OMP_TGT_MAPTYPE_FROM) { + void **ShadowHstPtrAddr = (void **)Itr->first; + *ShadowHstPtrAddr = Itr->second.HstPtrVal; + DP("Restoring original host pointer value " DPxMOD " for host " + "pointer " DPxMOD "\n", + DPxPTR(Itr->second.HstPtrVal), DPxPTR(ShadowHstPtrAddr)); + } + // If the struct is to be deallocated, remove the shadow entry. + if (Info.DelEntry) { + DP("Removing shadow pointer " DPxMOD "\n", DPxPTR((void **)Itr->first)); + auto OldItr = Itr; + Itr++; + Device->ShadowPtrMap.erase(OldItr); + } else { + ++Itr; + } + return OFFLOAD_SUCCESS; + }; + applyToShadowMapEntries(*Device, CB, Info.HstPtrBegin, Info.DataSize, + Info.TPR); + + // If we are deleting the entry the DataMapMtx is locked and we own + // the entry. + if (Info.DelEntry) { + if (!FromMapperBase || FromMapperBase != Info.HstPtrBegin) + Ret = Device->deallocTgtPtr(HDTTMap, LR, Info.DataSize); + + if (Ret != OFFLOAD_SUCCESS) { + REPORT("Deallocating data from device failed.\n"); + break; + } + } + } + + return Ret; +} + /// Internal function to undo the mapping and retrieve the data from the device. int targetDataEnd(ident_t *Loc, DeviceTy &Device, int32_t ArgNum, void **ArgBases, void **Args, int64_t *ArgSizes, int64_t *ArgTypes, map_var_info_t *ArgNames, void **ArgMappers, AsyncInfoTy &AsyncInfo, bool FromMapper) { - int Ret; + int Ret = OFFLOAD_SUCCESS; SmallVector PostProcessingPtrs; void *FromMapperBase = nullptr; // process each input. @@ -839,75 +936,13 @@ } } - // TODO: We should not synchronize here but pass the AsyncInfo object to the - // allocate/deallocate device APIs. - // - // We need to synchronize before deallocating data. - Ret = AsyncInfo.synchronize(); - if (Ret != OFFLOAD_SUCCESS) - return OFFLOAD_FAIL; - - // Deallocate target pointer - for (PostProcessingInfo &Info : PostProcessingPtrs) { - // If we marked the entry to be deleted we need to verify no other thread - // reused it by now. If deletion is still supposed to happen by this thread - // LR will be set and exclusive access to the HDTT map will avoid another - // thread reusing the entry now. Note that we do not request (exclusive) - // access to the HDTT map if Info.DelEntry is not set. - LookupResult LR; - DeviceTy::HDTTMapAccessorTy HDTTMap = - Device.HostDataToTargetMap.getExclusiveAccessor(!Info.DelEntry); - - if (Info.DelEntry) { - LR = Device.lookupMapping(HDTTMap, Info.HstPtrBegin, Info.DataSize); - if (LR.Entry->getTotalRefCount() != 0 || - LR.Entry->getDeleteThreadId() != std::this_thread::get_id()) { - // The thread is not in charge of deletion anymore. Give up access to - // the HDTT map and unset the deletion flag. - HDTTMap.destroy(); - Info.DelEntry = false; - } - } - - // If we copied back to the host a struct/array containing pointers, we - // need to restore the original host pointer values from their shadow - // copies. If the struct is going to be deallocated, remove any remaining - // shadow pointer entries for this struct. - auto CB = [&](ShadowPtrListTy::iterator &Itr) { - // If we copied the struct to the host, we need to restore the pointer. - if (Info.ArgType & OMP_TGT_MAPTYPE_FROM) { - void **ShadowHstPtrAddr = (void **)Itr->first; - *ShadowHstPtrAddr = Itr->second.HstPtrVal; - DP("Restoring original host pointer value " DPxMOD " for host " - "pointer " DPxMOD "\n", - DPxPTR(Itr->second.HstPtrVal), DPxPTR(ShadowHstPtrAddr)); - } - // If the struct is to be deallocated, remove the shadow entry. - if (Info.DelEntry) { - DP("Removing shadow pointer " DPxMOD "\n", DPxPTR((void **)Itr->first)); - auto OldItr = Itr; - Itr++; - Device.ShadowPtrMap.erase(OldItr); - } else { - ++Itr; - } - return OFFLOAD_SUCCESS; - }; - applyToShadowMapEntries(Device, CB, Info.HstPtrBegin, Info.DataSize, - Info.TPR); - - // If we are deleting the entry the DataMapMtx is locked and we own the - // entry. - if (Info.DelEntry) { - if (!FromMapperBase || FromMapperBase != Info.HstPtrBegin) - Ret = Device.deallocTgtPtr(HDTTMap, LR, Info.DataSize); - - if (Ret != OFFLOAD_SUCCESS) { - REPORT("Deallocating data from device failed.\n"); - break; - } - } - } + // Add post-processing functions + AsyncInfo.addPostProcessingFunction( + [=, Device = &Device, + PostProcessingPtrs = std::move(PostProcessingPtrs)]() mutable -> int { + return postProcessingTargetDataEnd(Device, PostProcessingPtrs, + FromMapperBase); + }); return Ret; } @@ -947,20 +982,22 @@ return OFFLOAD_FAIL; } - auto CB = [&](ShadowPtrListTy::iterator &Itr) { - void **ShadowHstPtrAddr = (void **)Itr->first; - // Wait for device-to-host memcopies for whole struct to complete, - // before restoring the correct host pointer. - if (AsyncInfo.synchronize() != OFFLOAD_SUCCESS) - return OFFLOAD_FAIL; - *ShadowHstPtrAddr = Itr->second.HstPtrVal; - DP("Restoring original host pointer value " DPxMOD - " for host pointer " DPxMOD "\n", - DPxPTR(Itr->second.HstPtrVal), DPxPTR(ShadowHstPtrAddr)); - ++Itr; + // Wait for device-to-host memcopies for whole struct to complete, + // before restoring the correct host pointer. + AsyncInfo.addPostProcessingFunction([=, Device = &Device]() -> int { + auto CB = [&](ShadowPtrListTy::iterator &Itr) { + void **ShadowHstPtrAddr = (void **)Itr->first; + *ShadowHstPtrAddr = Itr->second.HstPtrVal; + DP("Restoring original host pointer value " DPxMOD + " for host pointer " DPxMOD "\n", + DPxPTR(Itr->second.HstPtrVal), DPxPTR(ShadowHstPtrAddr)); + ++Itr; + return OFFLOAD_SUCCESS; + }; + applyToShadowMapEntries(*Device, CB, HstPtrBegin, ArgSize, TPR); + return OFFLOAD_SUCCESS; - }; - applyToShadowMapEntries(Device, CB, HstPtrBegin, ArgSize, TPR); + }); } if (ArgType & OMP_TGT_MAPTYPE_TO) { @@ -1157,19 +1194,19 @@ /// first-private arguments and transfer them all at once. struct FirstPrivateArgInfoTy { /// The index of the element in \p TgtArgs corresponding to the argument - const int Index; + int Index; /// Host pointer begin - const char *HstPtrBegin; + char *HstPtrBegin; /// Host pointer end - const char *HstPtrEnd; + char *HstPtrEnd; /// Aligned size - const int64_t AlignedSize; + int64_t AlignedSize; /// Host pointer name - const map_var_info_t HstPtrName = nullptr; + map_var_info_t HstPtrName = nullptr; - FirstPrivateArgInfoTy(int Index, const void *HstPtr, int64_t Size, + FirstPrivateArgInfoTy(int Index, void *HstPtr, int64_t Size, const map_var_info_t HstPtrName = nullptr) - : Index(Index), HstPtrBegin(reinterpret_cast(HstPtr)), + : Index(Index), HstPtrBegin(reinterpret_cast(HstPtr)), HstPtrEnd(HstPtrBegin + Size), AlignedSize(Size + Size % Alignment), HstPtrName(HstPtrName) {} }; @@ -1471,12 +1508,17 @@ return OFFLOAD_FAIL; } - // Free target memory for private arguments - Ret = PrivateArgumentManager.free(); - if (Ret != OFFLOAD_SUCCESS) { - REPORT("Failed to deallocate target memory for private args\n"); - return OFFLOAD_FAIL; - } + // Free target memory for private arguments after synchronization. + AsyncInfo.addPostProcessingFunction( + [PrivateArgumentManager = + std::move(PrivateArgumentManager)]() mutable -> int { + int Ret = PrivateArgumentManager.free(); + if (Ret != OFFLOAD_SUCCESS) { + REPORT("Failed to deallocate target memory for private args\n"); + return OFFLOAD_FAIL; + } + return Ret; + }); return OFFLOAD_SUCCESS; } @@ -1530,7 +1572,7 @@ PrivateArgumentManagerTy PrivateArgumentManager(Device, AsyncInfo); - int Ret; + int Ret = OFFLOAD_SUCCESS; if (ArgNum) { // Process data, such as data mapping, before launching the kernel Ret = processDataBefore(Loc, DeviceId, HostPtr, ArgNum, ArgBases, Args, diff --git a/openmp/libomptarget/src/private.h b/openmp/libomptarget/src/private.h --- a/openmp/libomptarget/src/private.h +++ b/openmp/libomptarget/src/private.h @@ -115,6 +115,10 @@ kmp_depend_info_t *dep_list, kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) __attribute__((weak)); +void *__kmpc_omp_get_target_async_handle(kmp_int32 gtid) __attribute__((weak)); +void __kmpc_omp_set_target_async_handle(kmp_int32 gtid, void *handle) + __attribute__((weak)); +bool __kmpc_omp_has_task_team(kmp_int32 gtid) __attribute__((weak)); #ifdef __cplusplus } #endif @@ -187,6 +191,55 @@ } } +class TaskAsyncInfoTy { + const int ExecThreadID = -1; + AsyncInfoTy::SyncType SyncType = AsyncInfoTy::SyncType::BLOCKING; + AsyncInfoTy *AsyncInfo = nullptr; + bool IsNew = false; + +public: + TaskAsyncInfoTy(DeviceTy &Device) + : ExecThreadID(__kmpc_global_thread_num(NULL)) { + // Acquire the AsyncInfo stored in async handle of the current task being + // executed. If no valid async handle is present, a new AsyncInfo is + // allocated and stored in the current task. + AsyncInfo = (AsyncInfoTy *)__kmpc_omp_get_target_async_handle(ExecThreadID); + + if (!AsyncInfo) { + AsyncInfo = new AsyncInfoTy(Device); + __kmpc_omp_set_target_async_handle(ExecThreadID, (void *)AsyncInfo); + IsNew = true; + } + + // Define the synchronization type based on the current task context. Only + // tasks with an assigned task team can be re-enqueue and thus can use the + // non-blocking synchronization scheme. + if (__kmpc_omp_has_task_team(ExecThreadID)) + SyncType = AsyncInfoTy::SyncType::NON_BLOCKING; + } + + /// Delete AsyncInfo if it is completed and remove it from the current task + /// async handle. + ~TaskAsyncInfoTy() { + if (!AsyncInfo || !AsyncInfo->isDone()) + return; + + delete AsyncInfo; + __kmpc_omp_set_target_async_handle(ExecThreadID, NULL); + } + + AsyncInfoTy &operator*() { return *AsyncInfo; } + + AsyncInfoTy::SyncType getSyncType() { return SyncType; } + + /// Return if the device side operations should be dispatched or not. When the + /// async info attached to the task struct was just created, the target region + /// operations must be dispatched and accumulated on the internal AsyncInfo. + /// Otherwise, the operations were already dispatched before and only + /// synchronization is needed. + bool shouldDispatch() { return !IsNew; } +}; + #ifdef OMPTARGET_PROFILE_ENABLED #include "llvm/Support/TimeProfiler.h" #define TIMESCOPE() llvm::TimeTraceScope TimeScope(__FUNCTION__) diff --git a/openmp/libomptarget/src/rtl.cpp b/openmp/libomptarget/src/rtl.cpp --- a/openmp/libomptarget/src/rtl.cpp +++ b/openmp/libomptarget/src/rtl.cpp @@ -212,6 +212,8 @@ *((void **)&R.run_team_region_async) = dlsym(DynlibHandle, "__tgt_rtl_run_target_team_region_async"); *((void **)&R.synchronize) = dlsym(DynlibHandle, "__tgt_rtl_synchronize"); + *((void **)&R.query_async) = + dlsym(DynlibHandle, "__tgt_rtl_query_async"); *((void **)&R.data_exchange) = dlsym(DynlibHandle, "__tgt_rtl_data_exchange"); *((void **)&R.data_exchange_async) = diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h --- a/openmp/runtime/src/kmp.h +++ b/openmp/runtime/src/kmp.h @@ -2475,6 +2475,10 @@ } kmp_tasking_flags_t; +typedef struct kmp_target_data { + void *async_handle; // libomptarget async handle for task completion query +} kmp_target_data_t; + struct kmp_taskdata { /* aligned during dynamic allocation */ kmp_int32 td_task_id; /* id, assigned by debugger */ kmp_tasking_flags_t td_flags; /* task flags */ @@ -2517,6 +2521,7 @@ #if OMPT_SUPPORT ompt_task_info_t ompt_task_info; #endif + kmp_target_data_t td_target_data; }; // struct kmp_taskdata // Make sure padding above worked @@ -4011,6 +4016,11 @@ KMP_EXPORT void __kmp_set_teams_thread_limit(int limit); KMP_EXPORT int __kmp_get_teams_thread_limit(void); +/* Interface target task integration */ +KMP_EXPORT void *__kmpc_omp_get_target_async_handle(kmp_int32 gtid); +KMP_EXPORT void __kmpc_omp_set_target_async_handle(kmp_int32 gtid, void *handle); +KMP_EXPORT bool __kmpc_omp_has_task_team(kmp_int32 gtid); + /* Lock interface routines (fast versions with gtid passed in) */ KMP_EXPORT void __kmpc_init_lock(ident_t *loc, kmp_int32 gtid, void **user_lock); diff --git a/openmp/runtime/src/kmp_tasking.cpp b/openmp/runtime/src/kmp_tasking.cpp --- a/openmp/runtime/src/kmp_tasking.cpp +++ b/openmp/runtime/src/kmp_tasking.cpp @@ -1063,7 +1063,7 @@ KMP_DEBUG_ASSERT(taskdata->td_flags.started == 1); KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0); - bool detach = false; + bool completed = true; if (UNLIKELY(taskdata->td_flags.detachable == TASK_DETACHABLE)) { if (taskdata->td_allow_completion_event.type == KMP_EVENT_ALLOW_COMPLETION) { @@ -1087,13 +1087,24 @@ // __kmp_fulfill_event might free taskdata at any time from now taskdata->td_flags.proxy = TASK_PROXY; // proxify! - detach = true; + completed = false; } __kmp_release_tas_lock(&taskdata->td_allow_completion_event.lock, gtid); } } - if (!detach) { + // Tasks with valid target async handles must be re-enqueued. + if (taskdata->td_target_data.async_handle != NULL) { + // Note: no need to translate gtid to its shadow. If the current thread is a + // hidden helper one, then the gtid is already correct. Otherwise, hidden + // helper threads are disabled, and gtid refers to a OpenMP thread. + __kmpc_give_task(task, __kmp_tid_from_gtid(gtid)); + if (KMP_HIDDEN_HELPER_THREAD(gtid)) + __kmp_hidden_helper_worker_thread_signal(); + completed = false; + } + + if (completed) { taskdata->td_flags.complete = 1; // mark the task as completed #if OMPT_SUPPORT @@ -1125,6 +1136,13 @@ // function KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 1); taskdata->td_flags.executing = 0; // suspend the finishing task + + // Decrement the counter of hidden helper tasks to be executed. + if (taskdata->td_flags.hidden_helper) { + // Hidden helper tasks can only be executed by hidden helper threads. + KMP_ASSERT(KMP_HIDDEN_HELPER_THREAD(gtid)); + KMP_ATOMIC_DEC(&__kmp_unexecuted_hidden_helper_tasks); + } } KA_TRACE( @@ -1136,7 +1154,7 @@ // johnmc: if an asynchronous inquiry peers into the runtime system // it doesn't see the freed task as the current task. thread->th.th_current_task = resumed_task; - if (!detach) + if (completed) __kmp_free_task_and_ancestors(gtid, taskdata, thread); // TODO: GEH - make sure root team implicit task is initialized properly. @@ -1532,6 +1550,7 @@ parent_task->td_taskgroup; // task inherits taskgroup from the parent task taskdata->td_dephash = NULL; taskdata->td_depnode = NULL; + taskdata->td_target_data.async_handle = NULL; if (flags->tiedness == TASK_UNTIED) taskdata->td_last_tied = NULL; // will be set when the task is scheduled else @@ -1674,13 +1693,6 @@ } #endif - // Decreament the counter of hidden helper tasks to be executed - if (taskdata->td_flags.hidden_helper) { - // Hidden helper tasks can only be executed by hidden helper threads - KMP_ASSERT(KMP_HIDDEN_HELPER_THREAD(gtid)); - KMP_ATOMIC_DEC(&__kmp_unexecuted_hidden_helper_tasks); - } - // Proxy tasks are not handled by the runtime if (taskdata->td_flags.proxy != TASK_PROXY) { __kmp_task_start(gtid, task, current_task); // OMPT only if not discarded @@ -5131,3 +5143,58 @@ modifier, task_dup); KA_TRACE(20, ("__kmpc_taskloop_5(exit): T#%d\n", gtid)); } + +/*! +@ingroup TASKING +@param gtid Global Thread ID of current thread +@return Returns a opaque pointer to the thread's current task. If no task is +present, returns NULL. + +Acqurires the target async handle from the current task. +*/ +void *__kmpc_omp_get_target_async_handle(kmp_int32 gtid) { + void *handle = NULL; + kmp_info_t *thread = __kmp_thread_from_gtid(gtid); + kmp_taskdata_t *taskdata = thread->th.th_current_task; + + if (taskdata) { + handle = taskdata->td_target_data.async_handle; + } + + return handle; +} + +/*! +@ingroup TASKING +@param gtid Global Thread ID of current thread +@param handle New async handle address + +Sets the target async handle for the current task. +*/ +void __kmpc_omp_set_target_async_handle(kmp_int32 gtid, void *handle) { + kmp_info_t *thread = __kmp_thread_from_gtid(gtid); + kmp_taskdata_t *taskdata = thread->th.th_current_task; + + if (taskdata) { + taskdata->td_target_data.async_handle = handle; + } +} + +/*! +@ingroup TASKING +@param gtid Global Thread ID of current thread +@return Returns true if the current thread of the given thread has a task team +allocated to it. + +Checks if the current thread has a task team. +*/ +KMP_EXPORT bool __kmpc_omp_has_task_team(kmp_int32 gtid) { + kmp_info_t *thread = __kmp_thread_from_gtid(gtid); + kmp_taskdata_t *taskdata = thread->th.th_current_task; + + if (taskdata) { + return taskdata->td_task_team != NULL; + } + + return FALSE; +}