diff --git a/openmp/libomptarget/include/OmptCallback.h b/openmp/libomptarget/include/OmptCallback.h --- a/openmp/libomptarget/include/OmptCallback.h +++ b/openmp/libomptarget/include/OmptCallback.h @@ -27,6 +27,13 @@ FOREACH_OMPT_NOEMI_EVENT(macro) \ FOREACH_OMPT_EMI_EVENT(macro) +#define performIfOmptInitialized(stmt) \ + do { \ + if (llvm::omp::target::ompt::Initialized) { \ + stmt; \ + } \ + } while (0) + #define performOmptCallback(CallbackName, ...) \ do { \ if (ompt_callback_##CallbackName##_fn) \ @@ -89,6 +96,8 @@ } // namespace omp } // namespace llvm +#else +#define performIfOmptInitialized(stmt) #endif // OMPT_SUPPORT #pragma pop_macro("DEBUG_PREFIX") diff --git a/openmp/libomptarget/src/OmptCallback.cpp b/openmp/libomptarget/src/OmptCallback.cpp --- a/openmp/libomptarget/src/OmptCallback.cpp +++ b/openmp/libomptarget/src/OmptCallback.cpp @@ -35,12 +35,20 @@ FOREACH_OMPT_EMI_EVENT(defineOmptCallback) #undef defineOmptCallback -/// Thread local state for target region and associated metadata -thread_local llvm::omp::target::ompt::Interface OmptInterface; +/// Forward declaration +class LibomptargetRtlFinalizer; -/// Define function pointers -ompt_get_task_data_t ompt_get_task_data_fn = nullptr; +/// Object that will maintain the RTL finalizer from the plugin +LibomptargetRtlFinalizer *LibraryFinalizer = nullptr; + +thread_local Interface llvm::omp::target::ompt::RegionInterface; + +bool llvm::omp::target::ompt::Initialized = false; + +ompt_get_callback_t llvm::omp::target::ompt::lookupCallbackByCode = nullptr; +ompt_function_lookup_t llvm::omp::target::ompt::lookupCallbackByName = nullptr; ompt_get_target_task_data_t ompt_get_target_task_data_fn = nullptr; +ompt_get_task_data_t ompt_get_task_data_fn = nullptr; /// Unique correlation id static std::atomic IdCounter(1); @@ -51,14 +59,14 @@ /// Create a new correlation id and update the operations id static uint64_t createOpId() { uint64_t NewId = createId(); - OmptInterface.setHostOpId(NewId); + RegionInterface.setHostOpId(NewId); return NewId; } /// Create a new correlation id and update the target region id static uint64_t createRegionId() { uint64_t NewId = createId(); - OmptInterface.setTargetDataValue(NewId); + RegionInterface.setTargetDataValue(NewId); return NewId; } @@ -230,6 +238,7 @@ numTeams); } } + void Interface::beginTargetDataEnter(int64_t DeviceId, void *Code) { beginTargetRegion(); if (ompt_callback_target_emi_fn) { @@ -391,14 +400,6 @@ llvm::SmallVector RtlFinalizationFunctions; }; -/// Object that will maintain the RTL finalizer from the plugin -LibomptargetRtlFinalizer *LibraryFinalizer = nullptr; - -bool llvm::omp::target::ompt::Initialized = false; - -ompt_get_callback_t llvm::omp::target::ompt::lookupCallbackByCode = nullptr; -ompt_function_lookup_t llvm::omp::target::ompt::lookupCallbackByName = nullptr; - int llvm::omp::target::ompt::initializeLibrary(ompt_function_lookup_t lookup, int initial_device_num, ompt_data_t *tool_data) { @@ -418,6 +419,9 @@ assert(lookupCallbackByCode && "lookupCallbackByCode should be non-null"); assert(lookupCallbackByName && "lookupCallbackByName should be non-null"); + assert(ompt_get_task_data_fn && "ompt_get_task_data_fn should be non-null"); + assert(ompt_get_target_task_data_fn && + "ompt_get_target_task_data_fn should be non-null"); assert(LibraryFinalizer == nullptr && "LibraryFinalizer should not be initialized yet"); @@ -434,6 +438,7 @@ // with this library LibraryFinalizer->finalize(); delete LibraryFinalizer; + Initialized = false; } void llvm::omp::target::ompt::connectLibrary() { diff --git a/openmp/libomptarget/src/OmptInterface.h b/openmp/libomptarget/src/OmptInterface.h --- a/openmp/libomptarget/src/OmptInterface.h +++ b/openmp/libomptarget/src/OmptInterface.h @@ -13,8 +13,14 @@ #ifndef _OMPTARGET_OMPTINTERFACE_H #define _OMPTARGET_OMPTINTERFACE_H +#include +#include + +#include "OmptCallback.h" #include "omp-tools.h" +#include "llvm/Support/ErrorHandling.h" + // If target OMPT support is compiled in #ifdef OMPT_SUPPORT #define OMPT_IF_BUILT(stmt) stmt @@ -22,6 +28,8 @@ #define OMPT_IF_BUILT(stmt) #endif +#define OMPT_GET_RETURN_ADDRESS(level) __builtin_return_address(level) + /// Callbacks for target regions require task_data representing the /// encountering task. /// Callbacks for target regions and target data ops require @@ -108,6 +116,66 @@ /// Top-level function for invoking callback after target construct void endTarget(int64_t DeviceId, void *Code); + // Functionpair getter: Target data operations + template auto getCallbacks() { + if constexpr (OpType == ompt_target_data_alloc || + OpType == ompt_target_data_alloc_async) + return std::make_pair(std::mem_fn(&Interface::beginTargetDataAlloc), + std::mem_fn(&Interface::endTargetDataAlloc)); + + if constexpr (OpType == ompt_target_data_delete || + OpType == ompt_target_data_delete_async) + return std::make_pair(std::mem_fn(&Interface::beginTargetDataDelete), + std::mem_fn(&Interface::endTargetDataDelete)); + + if constexpr (OpType == ompt_target_data_transfer_to_device || + OpType == ompt_target_data_transfer_to_device_async) + return std::make_pair(std::mem_fn(&Interface::beginTargetDataSubmit), + std::mem_fn(&Interface::endTargetDataSubmit)); + + if constexpr (OpType == ompt_target_data_transfer_from_device || + OpType == ompt_target_data_transfer_from_device_async) + return std::make_pair(std::mem_fn(&Interface::beginTargetDataRetrieve), + std::mem_fn(&Interface::endTargetDataRetrieve)); + + llvm_unreachable("Unhandled target data operation type!"); + } + + // Target region operations + template auto getCallbacks() { + if constexpr (OpType == ompt_target_enter_data || + OpType == ompt_target_enter_data_nowait) + return std::make_pair(std::mem_fn(&Interface::beginTargetDataEnter), + std::mem_fn(&Interface::endTargetDataEnter)); + + if constexpr (OpType == ompt_target_exit_data || + OpType == ompt_target_exit_data_nowait) + return std::make_pair(std::mem_fn(&Interface::beginTargetDataExit), + std::mem_fn(&Interface::endTargetDataExit)); + + if constexpr (OpType == ompt_target_update || + OpType == ompt_target_update_nowait) + return std::make_pair(std::mem_fn(&Interface::beginTargetUpdate), + std::mem_fn(&Interface::endTargetUpdate)); + + if constexpr (OpType == ompt_target || OpType == ompt_target_nowait) + return std::make_pair(std::mem_fn(&Interface::beginTarget), + std::mem_fn(&Interface::endTarget)); + + llvm_unreachable("Unknown target region operation type!"); + } + + // Kernel launch operation + template auto getCallbacks() { + // We use 'ompt_callbacks_t', because no other enum is currently available + // to model a kernel launch / target submit operation. + if constexpr (OpType == ompt_callback_target_submit) + return std::make_pair(std::mem_fn(&Interface::beginTargetSubmit), + std::mem_fn(&Interface::endTargetSubmit)); + + llvm_unreachable("Unhandled target operation!"); + } + /// Setters for target region and target operation correlation ids void setTargetDataValue(uint64_t DataValue) { TargetData.value = DataValue; } void setTargetDataPtr(void *DataPtr) { TargetData.ptr = DataPtr; } @@ -147,11 +215,45 @@ void endTargetRegion(); }; +/// Thread local state for target region and associated metadata +extern thread_local Interface RegionInterface; + +template +void InvokeInterfaceFunction(FuncTy Func, ArgsTy Args, + std::index_sequence) { + std::invoke(Func, RegionInterface, std::get(Args)...); +} + +template class InterfaceRAII { +public: + InterfaceRAII(FunctionPairTy FunctionPair, ArgsTy... Args) + : Arguments(Args...), beginFunction(std::get<0>(FunctionPair)), + endFunction(std::get<1>(FunctionPair)) { + performIfOmptInitialized(begin()); + } + ~InterfaceRAII() { performIfOmptInitialized(end()); } + +private: + void begin() { + auto IndexSequence = + std::make_index_sequence>{}; + InvokeInterfaceFunction(beginFunction, Arguments, IndexSequence); + } + + void end() { + auto IndexSequence = + std::make_index_sequence>{}; + InvokeInterfaceFunction(endFunction, Arguments, IndexSequence); + } + + std::tuple Arguments; + typename FunctionPairTy::first_type beginFunction; + typename FunctionPairTy::second_type endFunction; +}; + } // namespace ompt } // namespace target } // namespace omp } // namespace llvm -extern thread_local llvm::omp::target::ompt::Interface OmptInterface; - #endif // _OMPTARGET_OMPTINTERFACE_H diff --git a/openmp/libomptarget/src/device.cpp b/openmp/libomptarget/src/device.cpp --- a/openmp/libomptarget/src/device.cpp +++ b/openmp/libomptarget/src/device.cpp @@ -11,6 +11,8 @@ //===----------------------------------------------------------------------===// #include "device.h" +#include "OmptCallback.h" +#include "OmptInterface.h" #include "omptarget.h" #include "private.h" #include "rtl.h" @@ -23,6 +25,8 @@ #include #include +using namespace llvm::omp::target::ompt; + int HostDataToTargetTy::addEventIfNecessary(DeviceTy &Device, AsyncInfoTy &AsyncInfo) const { // First, check if the user disabled atomic map transfer/malloc/dealloc. @@ -554,10 +558,22 @@ } void *DeviceTy::allocData(int64_t Size, void *HstPtr, int32_t Kind) { + /// RAII to establish tool anchors before and after data allocation + InterfaceRAII TargetDataAllocRAII( + RegionInterface.getCallbacks(), RTLDeviceID, + HstPtr, Size, + /* CodePtr */ OMPT_GET_RETURN_ADDRESS(0)); + return RTL->data_alloc(RTLDeviceID, Size, HstPtr, Kind); } int32_t DeviceTy::deleteData(void *TgtAllocBegin, int32_t Kind) { + /// RAII to establish tool anchors before and after data deletion + InterfaceRAII TargetDataDeleteRAII( + RegionInterface.getCallbacks(), RTLDeviceID, + TgtAllocBegin, + /* CodePtr */ OMPT_GET_RETURN_ADDRESS(0)); + return RTL->data_delete(RTLDeviceID, TgtAllocBegin, Kind); } @@ -589,6 +605,12 @@ Entry); } + /// RAII to establish tool anchors before and after data submit + InterfaceRAII TargetDataSubmitRAII( + RegionInterface.getCallbacks(), + RTLDeviceID, TgtPtrBegin, HstPtrBegin, Size, + /* CodePtr */ OMPT_GET_RETURN_ADDRESS(0)); + if (!AsyncInfo || !RTL->data_submit_async || !RTL->synchronize) return RTL->data_submit(RTLDeviceID, TgtPtrBegin, HstPtrBegin, Size); return RTL->data_submit_async(RTLDeviceID, TgtPtrBegin, HstPtrBegin, Size, @@ -610,6 +632,12 @@ Entry); } + /// RAII to establish tool anchors before and after data retrieval + InterfaceRAII TargetDataRetrieveRAII( + RegionInterface.getCallbacks(), + RTLDeviceID, HstPtrBegin, TgtPtrBegin, Size, + /* CodePtr */ OMPT_GET_RETURN_ADDRESS(0)); + if (!RTL->data_retrieve_async || !RTL->synchronize) return RTL->data_retrieve(RTLDeviceID, HstPtrBegin, TgtPtrBegin, Size); return RTL->data_retrieve_async(RTLDeviceID, HstPtrBegin, TgtPtrBegin, Size, diff --git a/openmp/libomptarget/src/interface.cpp b/openmp/libomptarget/src/interface.cpp --- a/openmp/libomptarget/src/interface.cpp +++ b/openmp/libomptarget/src/interface.cpp @@ -11,6 +11,8 @@ // //===----------------------------------------------------------------------===// +#include "OmptCallback.h" +#include "OmptInterface.h" #include "device.h" #include "omptarget.h" #include "private.h" @@ -24,6 +26,8 @@ #include #include +using namespace llvm::omp::target::ompt; + //////////////////////////////////////////////////////////////////////////////// /// adds requires flags EXTERN void __tgt_register_requires(int64_t Flags) { @@ -123,6 +127,11 @@ map_var_info_t *ArgNames, void **ArgMappers) { TIMESCOPE_WITH_IDENT(Loc); + /// RAII to establish tool anchors before and after data begin + InterfaceRAII TargetDataEnterRAII( + RegionInterface.getCallbacks(), DeviceId, + /* CodePtr */ OMPT_GET_RETURN_ADDRESS(0)); + targetDataMapper(Loc, DeviceId, ArgNum, ArgsBase, Args, ArgSizes, ArgTypes, ArgNames, ArgMappers, targetDataBegin, "Entering OpenMP data region", "begin"); @@ -149,6 +158,11 @@ map_var_info_t *ArgNames, void **ArgMappers) { TIMESCOPE_WITH_IDENT(Loc); + /// RAII to establish tool anchors before and after data end + InterfaceRAII TargetDataExitRAII( + RegionInterface.getCallbacks(), DeviceId, + /* CodePtr */ OMPT_GET_RETURN_ADDRESS(0)); + targetDataMapper(Loc, DeviceId, ArgNum, ArgsBase, Args, ArgSizes, ArgTypes, ArgNames, ArgMappers, targetDataEnd, "Exiting OpenMP data region", "end"); @@ -172,6 +186,11 @@ map_var_info_t *ArgNames, void **ArgMappers) { TIMESCOPE_WITH_IDENT(Loc); + /// RAII to establish tool anchors before and after data update + InterfaceRAII TargetDataUpdateRAII( + RegionInterface.getCallbacks(), DeviceId, + /* CodePtr */ OMPT_GET_RETURN_ADDRESS(0)); + targetDataMapper( Loc, DeviceId, ArgNum, ArgsBase, Args, ArgSizes, ArgTypes, ArgNames, ArgMappers, targetDataUpdate, "Updating OpenMP data", "update"); @@ -271,6 +290,9 @@ DeviceTy &Device = *PM->Devices[DeviceId]; TargetAsyncInfoTy TargetAsyncInfo(Device); AsyncInfoTy &AsyncInfo = TargetAsyncInfo; + InterfaceRAII TargetDataAllocRAII(RegionInterface.getCallbacks(), + DeviceId, + /* CodePtr */ OMPT_GET_RETURN_ADDRESS(0)); int Rc = OFFLOAD_SUCCESS; Rc = target(Loc, Device, HostPtr, *KernelArgs, AsyncInfo); @@ -336,6 +358,9 @@ return OMP_TGT_FAIL; } DeviceTy &Device = *PM->Devices[DeviceId]; + InterfaceRAII TargetDataAllocRAII(RegionInterface.getCallbacks(), + DeviceId, + /* CodePtr */ OMPT_GET_RETURN_ADDRESS(0)); AsyncInfoTy AsyncInfo(Device); int Rc = target_replay(Loc, Device, HostPtr, DeviceMemory, DeviceMemorySize, diff --git a/openmp/libomptarget/src/omptarget.cpp b/openmp/libomptarget/src/omptarget.cpp --- a/openmp/libomptarget/src/omptarget.cpp +++ b/openmp/libomptarget/src/omptarget.cpp @@ -12,6 +12,8 @@ //===----------------------------------------------------------------------===// #include "omptarget.h" +#include "OmptCallback.h" +#include "OmptInterface.h" #include "device.h" #include "private.h" #include "rtl.h" @@ -24,6 +26,7 @@ #include using llvm::SmallVector; +using namespace llvm::omp::target::ompt; int AsyncInfoTy::synchronize() { int Result = OFFLOAD_SUCCESS; @@ -1670,6 +1673,16 @@ { assert(KernelArgs.NumArgs == TgtArgs.size() && "Argument count mismatch!"); TIMESCOPE_WITH_NAME_AND_IDENT("Initiate Kernel Launch", Loc); + +#ifdef OMPT_SUPPORT + assert(KernelArgs.NumTeams[1] == 0 && KernelArgs.NumTeams[2] == 0 && + "Multi dimensional launch not supported yet."); + /// RAII to establish tool anchors before and after kernel launch + int32_t NumTeams = KernelArgs.NumTeams[0]; + InterfaceRAII TargetSubmitRAII( + RegionInterface.getCallbacks(), NumTeams); +#endif + Ret = Device.launchKernel(TgtEntryPtr, TgtArgs.data(), TgtOffsets.data(), KernelArgs, AsyncInfo); } diff --git a/openmp/libomptarget/test/ompt/veccopy.c b/openmp/libomptarget/test/ompt/veccopy.c --- a/openmp/libomptarget/test/ompt/veccopy.c +++ b/openmp/libomptarget/test/ompt/veccopy.c @@ -55,10 +55,29 @@ return rc; } -/// CHECK: Could not register callback 'ompt_callback_target_data_op' -/// CHECK: Could not register callback 'ompt_callback_target' -/// CHECK: Could not register callback 'ompt_callback_target_submit' - /// CHECK: Callback Init: /// CHECK: Callback Load: +/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2 + +/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2 /// CHECK: Callback Fini: diff --git a/openmp/libomptarget/test/ompt/veccopy_disallow_both.c b/openmp/libomptarget/test/ompt/veccopy_disallow_both.c --- a/openmp/libomptarget/test/ompt/veccopy_disallow_both.c +++ b/openmp/libomptarget/test/ompt/veccopy_disallow_both.c @@ -58,10 +58,44 @@ return rc; } -/// CHECK: Could not register callback 'ompt_callback_target_data_op' -/// CHECK: Could not register callback 'ompt_callback_target' -/// CHECK: Could not register callback 'ompt_callback_target_submit' - /// CHECK: Callback Init: /// CHECK: Callback Load: +/// CHECK: Callback Target EMI: kind=1 endpoint=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback Target EMI: kind=1 endpoint=2 +/// CHECK: Callback Target EMI: kind=1 endpoint=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback Target EMI: kind=1 endpoint=2 /// CHECK: Callback Fini: diff --git a/openmp/libomptarget/test/ompt/veccopy_emi.c b/openmp/libomptarget/test/ompt/veccopy_emi.c --- a/openmp/libomptarget/test/ompt/veccopy_emi.c +++ b/openmp/libomptarget/test/ompt/veccopy_emi.c @@ -56,10 +56,46 @@ return rc; } -/// CHECK: Could not register callback 'ompt_callback_target_data_op_emi' -/// CHECK: Could not register callback 'ompt_callback_target_emi' -/// CHECK: Could not register callback 'ompt_callback_target_submit_emi' - /// CHECK: Callback Init: /// CHECK: Callback Load: +/// CHECK: Callback Target EMI: kind=1 endpoint=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback Submit EMI: endpoint=1 req_num_teams=1 +/// CHECK: Callback Submit EMI: endpoint=2 req_num_teams=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback Target EMI: kind=1 endpoint=2 +/// CHECK: Callback Target EMI: kind=1 endpoint=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback Submit EMI: endpoint=1 req_num_teams=0 +/// CHECK: Callback Submit EMI: endpoint=2 req_num_teams=0 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback Target EMI: kind=1 endpoint=2 /// CHECK: Callback Fini: diff --git a/openmp/libomptarget/test/ompt/veccopy_emi_map.c b/openmp/libomptarget/test/ompt/veccopy_emi_map.c --- a/openmp/libomptarget/test/ompt/veccopy_emi_map.c +++ b/openmp/libomptarget/test/ompt/veccopy_emi_map.c @@ -56,11 +56,47 @@ return rc; } -/// CHECK: Could not register callback 'ompt_callback_target_data_op_emi' -/// CHECK: Could not register callback 'ompt_callback_target_emi' -/// CHECK: Could not register callback 'ompt_callback_target_submit_emi' -/// CHECK: Could not register callback 'ompt_callback_target_map_emi' - +/// CHECK: 0: Could not register callback 'ompt_callback_target_map_emi' /// CHECK: Callback Init: /// CHECK: Callback Load: +/// CHECK: Callback Target EMI: kind=1 endpoint=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback Submit EMI: endpoint=1 req_num_teams=1 +/// CHECK: Callback Submit EMI: endpoint=2 req_num_teams=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback Target EMI: kind=1 endpoint=2 +/// CHECK: Callback Target EMI: kind=1 endpoint=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback Submit EMI: endpoint=1 req_num_teams=0 +/// CHECK: Callback Submit EMI: endpoint=2 req_num_teams=0 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback Target EMI: kind=1 endpoint=2 /// CHECK: Callback Fini: diff --git a/openmp/libomptarget/test/ompt/veccopy_map.c b/openmp/libomptarget/test/ompt/veccopy_map.c --- a/openmp/libomptarget/test/ompt/veccopy_map.c +++ b/openmp/libomptarget/test/ompt/veccopy_map.c @@ -55,10 +55,31 @@ return rc; } -/// CHECK: Could not register callback 'ompt_callback_target_data_op' -/// CHECK: Could not register callback 'ompt_callback_target' -/// CHECK: Could not register callback 'ompt_callback_target_submit' +/// CHECK: 0: Could not register callback 'ompt_callback_target_map' /// CHECK: Callback Init: /// CHECK: Callback Load: +/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2 + +/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2 /// CHECK: Callback Fini: diff --git a/openmp/libomptarget/test/ompt/veccopy_no_device_init.c b/openmp/libomptarget/test/ompt/veccopy_no_device_init.c --- a/openmp/libomptarget/test/ompt/veccopy_no_device_init.c +++ b/openmp/libomptarget/test/ompt/veccopy_no_device_init.c @@ -54,10 +54,29 @@ return rc; } -/// CHECK: Could not register callback 'ompt_callback_target_data_op' -/// CHECK: Could not register callback 'ompt_callback_target' -/// CHECK: Could not register callback 'ompt_callback_target_submit' - /// CHECK-NOT: Callback Init: /// CHECK-NOT: Callback Load: +/// CHECK-NOT: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK-NOT: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK-NOT: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2 + +/// CHECK-NOT: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK-NOT: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK-NOT: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2 /// CHECK-NOT: Callback Fini: diff --git a/openmp/libomptarget/test/ompt/veccopy_wrong_return.c b/openmp/libomptarget/test/ompt/veccopy_wrong_return.c --- a/openmp/libomptarget/test/ompt/veccopy_wrong_return.c +++ b/openmp/libomptarget/test/ompt/veccopy_wrong_return.c @@ -54,10 +54,29 @@ return rc; } -/// CHECK: Could not register callback 'ompt_callback_target_data_op' -/// CHECK: Could not register callback 'ompt_callback_target' -/// CHECK: Could not register callback 'ompt_callback_target_submit' - /// CHECK-NOT: Callback Init: /// CHECK-NOT: Callback Load: -/// CHECK-NOT: Callback Fini: +/// CHECK-NOT: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK-NOT: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK-NOT: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2 + +/// CHECK-NOT: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK-NOT: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK-NOT: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2 +/// CHECK-NOT: Callback Fini diff --git a/openmp/runtime/src/ompt-event-specific.h b/openmp/runtime/src/ompt-event-specific.h --- a/openmp/runtime/src/ompt-event-specific.h +++ b/openmp/runtime/src/ompt-event-specific.h @@ -55,13 +55,12 @@ #define ompt_callback_implicit_task_implemented ompt_event_MAY_ALWAYS -#define ompt_callback_target_implemented ompt_event_UNIMPLEMENTED -#define ompt_callback_target_emi_implemented ompt_event_UNIMPLEMENTED -#define ompt_callback_target_data_op_implemented ompt_event_UNIMPLEMENTED -#define ompt_callback_target_data_op_emi_implemented ompt_event_UNIMPLEMENTED -#define ompt_callback_target_submit_implemented ompt_event_UNIMPLEMENTED -#define ompt_callback_target_submit_emi_implemented ompt_event_UNIMPLEMENTED - +#define ompt_callback_target_implemented ompt_event_MAY_ALWAYS +#define ompt_callback_target_emi_implemented ompt_event_MAY_ALWAYS +#define ompt_callback_target_data_op_implemented ompt_event_MAY_ALWAYS +#define ompt_callback_target_data_op_emi_implemented ompt_event_MAY_ALWAYS +#define ompt_callback_target_submit_implemented ompt_event_MAY_ALWAYS +#define ompt_callback_target_submit_emi_implemented ompt_event_MAY_ALWAYS #define ompt_callback_control_tool_implemented ompt_event_MAY_ALWAYS #define ompt_callback_device_initialize_implemented ompt_event_MAY_ALWAYS