diff --git a/openmp/libomptarget/include/OmptCallback.h b/openmp/libomptarget/include/OmptCallback.h --- a/openmp/libomptarget/include/OmptCallback.h +++ b/openmp/libomptarget/include/OmptCallback.h @@ -27,6 +27,13 @@ FOREACH_OMPT_NOEMI_EVENT(macro) \ FOREACH_OMPT_EMI_EVENT(macro) +#define performIfOmptInitialized(stmt) \ + do { \ + if (llvm::omp::target::ompt::Initialized) { \ + stmt; \ + } \ + } while (0) + #define performOmptCallback(CallbackName, ...) \ do { \ if (ompt_callback_##CallbackName##_fn) \ @@ -81,11 +88,16 @@ /// functions to their respective higher layer. void connectLibrary(); +/// Status bit; true if OMPT has been successfully initialized. False otherwise. +extern bool Initialized; + } // namespace ompt } // namespace target } // namespace omp } // namespace llvm +#else +#define performIfOmptInitialized(stmt) #endif // OMPT_SUPPORT #pragma pop_macro("DEBUG_PREFIX") diff --git a/openmp/libomptarget/src/OmptCallback.cpp b/openmp/libomptarget/src/OmptCallback.cpp --- a/openmp/libomptarget/src/OmptCallback.cpp +++ b/openmp/libomptarget/src/OmptCallback.cpp @@ -35,13 +35,15 @@ FOREACH_OMPT_EMI_EVENT(defineOmptCallback) #undef defineOmptCallback -/// Thread local state for target region and associated metadata -thread_local llvm::omp::target::ompt::Interface OmptInterface; +thread_local Interface llvm::omp::target::ompt::RegionInterface; /// Define function pointers ompt_get_task_data_t ompt_get_task_data_fn = nullptr; ompt_get_target_task_data_t ompt_get_target_task_data_fn = nullptr; +/// Initialization status +bool llvm::omp::target::ompt::Initialized = false; + /// Unique correlation id static std::atomic IdCounter(1); @@ -51,14 +53,14 @@ /// Create a new correlation id and update the operations id static uint64_t createOpId() { uint64_t NewId = createId(); - OmptInterface.setHostOpId(NewId); + RegionInterface.setHostOpId(NewId); return NewId; } /// Create a new correlation id and update the target region id static uint64_t createRegionId() { uint64_t NewId = createId(); - OmptInterface.setTargetDataValue(NewId); + RegionInterface.setTargetDataValue(NewId); return NewId; } @@ -230,6 +232,7 @@ numTeams); } } + void Interface::beginTargetDataEnter(int64_t DeviceId, void *Code) { beginTargetRegion(); if (ompt_callback_target_emi_fn) { @@ -416,10 +419,14 @@ assert(lookupCallbackByCode && "lookupCallbackByCode should be non-null"); assert(lookupCallbackByName && "lookupCallbackByName should be non-null"); + assert(ompt_get_task_data_fn && "ompt_get_task_data_fn should be non-null"); + assert(ompt_get_target_task_data_fn && + "ompt_get_target_task_data_fn should be non-null"); assert(LibraryFinalizer == nullptr && "LibraryFinalizer should not be initialized yet"); LibraryFinalizer = new LibomptargetRtlFinalizer(); + Initialized = true; return 0; } @@ -430,6 +437,7 @@ // with this library LibraryFinalizer->finalize(); delete LibraryFinalizer; + Initialized = false; } void llvm::omp::target::ompt::connectLibrary() { diff --git a/openmp/libomptarget/src/OmptInterface.h b/openmp/libomptarget/src/OmptInterface.h --- a/openmp/libomptarget/src/OmptInterface.h +++ b/openmp/libomptarget/src/OmptInterface.h @@ -13,8 +13,11 @@ #ifndef _OMPTARGET_OMPTINTERFACE_H #define _OMPTARGET_OMPTINTERFACE_H +#include "OmptCallback.h" #include "omp-tools.h" +#include "stdio.h" + // If target OMPT support is compiled in #ifdef OMPT_SUPPORT #define OMPT_IF_BUILT(stmt) stmt @@ -22,6 +25,8 @@ #define OMPT_IF_BUILT(stmt) #endif +#define OMPT_GET_RETURN_ADDRESS(level) __builtin_return_address(level) + /// Callbacks for target regions require task_data representing the /// encountering task. /// Callbacks for target regions and target data ops require @@ -34,6 +39,8 @@ namespace target { namespace ompt { +enum class InterfaceOpType { Target, TargetData, TargetSubmit }; + /// Function pointers that will be used to track task_data and /// target_task_data. static ompt_get_task_data_t ompt_get_task_data_fn; @@ -147,11 +154,12 @@ void endTargetRegion(); }; +/// Thread local state for target region and associated metadata +extern thread_local Interface RegionInterface; + } // namespace ompt } // namespace target } // namespace omp } // namespace llvm -extern thread_local llvm::omp::target::ompt::Interface OmptInterface; - #endif // _OMPTARGET_OMPTINTERFACE_H diff --git a/openmp/libomptarget/src/device.cpp b/openmp/libomptarget/src/device.cpp --- a/openmp/libomptarget/src/device.cpp +++ b/openmp/libomptarget/src/device.cpp @@ -11,6 +11,8 @@ //===----------------------------------------------------------------------===// #include "device.h" +#include "OmptCallback.h" +#include "OmptInterface.h" #include "omptarget.h" #include "private.h" #include "rtl.h" @@ -23,6 +25,78 @@ #include #include +using namespace llvm::omp::target::ompt; + +/// RAII used to invoke callbacks before and after data ops +struct OmptInterfaceTargetDataOpRAII { + OmptInterfaceTargetDataOpRAII(int32_t Id, int64_t Sz, void *HPtr, void *TPtr, + ompt_target_data_op_t Op) + : DeviceId{Id}, Size{Sz}, CodePtr{nullptr}, HostPtr{HPtr}, TgtPtr{TPtr}, + TgtDataOp{Op} { + performIfOmptInitialized(beginOp()); + } + ~OmptInterfaceTargetDataOpRAII() { performIfOmptInitialized(endOp()); } + +private: + int32_t DeviceId; // Target device + int64_t Size; // Size of data transfer + void *CodePtr; // Return address + void *HostPtr; // Host data ptr + void *TgtPtr; // Target data ptr + ompt_target_data_op_t TgtDataOp; // Data transfer type + void beginOp() { + // if (!OmptEnabled) return; + CodePtr = OMPT_GET_RETURN_ADDRESS(0); + switch (TgtDataOp) { + case ompt_target_data_alloc: + case ompt_target_data_alloc_async: + RegionInterface.beginTargetDataAlloc(DeviceId, HostPtr, Size, CodePtr); + break; + case ompt_target_data_delete: + case ompt_target_data_delete_async: + RegionInterface.beginTargetDataDelete(DeviceId, TgtPtr, CodePtr); + break; + case ompt_target_data_transfer_to_device: + case ompt_target_data_transfer_to_device_async: + RegionInterface.beginTargetDataSubmit(DeviceId, TgtPtr, HostPtr, Size, + CodePtr); + break; + case ompt_target_data_transfer_from_device: + case ompt_target_data_transfer_from_device_async: + RegionInterface.beginTargetDataRetrieve(DeviceId, HostPtr, TgtPtr, Size, + CodePtr); + break; + default: + break; + } + } + void endOp() { + // if (!OmptEnabled) return; + switch (TgtDataOp) { + case ompt_target_data_alloc: + case ompt_target_data_alloc_async: + RegionInterface.endTargetDataAlloc(DeviceId, HostPtr, Size, CodePtr); + break; + case ompt_target_data_delete: + case ompt_target_data_delete_async: + RegionInterface.endTargetDataDelete(DeviceId, TgtPtr, CodePtr); + break; + case ompt_target_data_transfer_to_device: + case ompt_target_data_transfer_to_device_async: + RegionInterface.endTargetDataSubmit(DeviceId, TgtPtr, HostPtr, Size, + CodePtr); + break; + case ompt_target_data_transfer_from_device: + case ompt_target_data_transfer_from_device_async: + RegionInterface.endTargetDataRetrieve(DeviceId, HostPtr, TgtPtr, Size, + CodePtr); + break; + default: + break; + } + } +}; + int HostDataToTargetTy::addEventIfNecessary(DeviceTy &Device, AsyncInfoTy &AsyncInfo) const { // First, check if the user disabled atomic map transfer/malloc/dealloc. @@ -554,10 +628,17 @@ } void *DeviceTy::allocData(int64_t Size, void *HstPtr, int32_t Kind) { + /// RAII to establish tool anchors before and after data allocation + OmptInterfaceTargetDataOpRAII TgtDataAlloc( + RTLDeviceID, Size, HstPtr, nullptr /* TgtPtr */, ompt_target_data_alloc); return RTL->data_alloc(RTLDeviceID, Size, HstPtr, Kind); } int32_t DeviceTy::deleteData(void *TgtAllocBegin, int32_t Kind) { + /// RAII to establish tool anchors before and after data deletion + OmptInterfaceTargetDataOpRAII TgtDataDelete( + RTLDeviceID, 0 /* Size */, nullptr /* HostPtr */, TgtAllocBegin, + ompt_target_data_delete); return RTL->data_delete(RTLDeviceID, TgtAllocBegin, Kind); } @@ -589,6 +670,11 @@ Entry); } + /// RAII to establish tool anchors before and after data submit + OmptInterfaceTargetDataOpRAII TargetDataSubmitRAII( + RTLDeviceID, Size, HstPtrBegin, TgtPtrBegin, + ompt_target_data_transfer_to_device); + if (!AsyncInfo || !RTL->data_submit_async || !RTL->synchronize) return RTL->data_submit(RTLDeviceID, TgtPtrBegin, HstPtrBegin, Size); return RTL->data_submit_async(RTLDeviceID, TgtPtrBegin, HstPtrBegin, Size, @@ -610,6 +696,11 @@ Entry); } + /// RAII to establish tool anchors before and after data retrieval + OmptInterfaceTargetDataOpRAII TargetDataRetrieve( + RTLDeviceID, Size, HstPtrBegin, TgtPtrBegin, + ompt_target_data_transfer_from_device); + if (!RTL->data_retrieve_async || !RTL->synchronize) return RTL->data_retrieve(RTLDeviceID, HstPtrBegin, TgtPtrBegin, Size); return RTL->data_retrieve_async(RTLDeviceID, HstPtrBegin, TgtPtrBegin, Size, diff --git a/openmp/libomptarget/src/interface.cpp b/openmp/libomptarget/src/interface.cpp --- a/openmp/libomptarget/src/interface.cpp +++ b/openmp/libomptarget/src/interface.cpp @@ -11,6 +11,8 @@ // //===----------------------------------------------------------------------===// +#include "OmptCallback.h" +#include "OmptInterface.h" #include "device.h" #include "omptarget.h" #include "private.h" @@ -24,6 +26,65 @@ #include #include +using namespace llvm::omp::target::ompt; + +/// RAII used to invoke callbacks before and after target regions +struct OmptInterfaceTargetRAII { + OmptInterfaceTargetRAII(int64_t Id, ompt_target_t Op) + : CodePtr{nullptr}, DeviceId{Id}, TgtOp{Op} { + performIfOmptInitialized(beginOp()); + } + ~OmptInterfaceTargetRAII() { performIfOmptInitialized(endOp()); } + +private: + void *CodePtr; // Return address + int64_t DeviceId; // Target device + ompt_target_t TgtOp; // Target operation + void beginOp() { + // if (!OmptEnabled) return; + CodePtr = OMPT_GET_RETURN_ADDRESS(0); + switch (TgtOp) { + case ompt_target_enter_data: + case ompt_target_enter_data_nowait: + RegionInterface.beginTargetDataEnter(DeviceId, CodePtr); + break; + case ompt_target_exit_data: + case ompt_target_exit_data_nowait: + RegionInterface.beginTargetDataExit(DeviceId, CodePtr); + break; + case ompt_target_update: + case ompt_target_update_nowait: + RegionInterface.beginTargetUpdate(DeviceId, CodePtr); + break; + case ompt_target: + case ompt_target_nowait: + RegionInterface.beginTarget(DeviceId, CodePtr); + break; + } + } + void endOp() { + // if (!OmptEnabled) return; + switch (TgtOp) { + case ompt_target_enter_data: + case ompt_target_enter_data_nowait: + RegionInterface.endTargetDataEnter(DeviceId, CodePtr); + break; + case ompt_target_exit_data: + case ompt_target_exit_data_nowait: + RegionInterface.endTargetDataExit(DeviceId, CodePtr); + break; + case ompt_target_update: + case ompt_target_update_nowait: + RegionInterface.endTargetUpdate(DeviceId, CodePtr); + break; + case ompt_target: + case ompt_target_nowait: + RegionInterface.endTarget(DeviceId, CodePtr); + break; + } + } +}; + //////////////////////////////////////////////////////////////////////////////// /// adds requires flags EXTERN void __tgt_register_requires(int64_t Flags) { @@ -123,6 +184,8 @@ map_var_info_t *ArgNames, void **ArgMappers) { TIMESCOPE_WITH_IDENT(Loc); + /// RAII to establish tool anchors before and after data begin + OmptInterfaceTargetRAII TargetDataBeginRAII(DeviceId, ompt_target_enter_data); targetDataMapper(Loc, DeviceId, ArgNum, ArgsBase, Args, ArgSizes, ArgTypes, ArgNames, ArgMappers, targetDataBegin, "Entering OpenMP data region", "begin"); @@ -149,6 +212,8 @@ map_var_info_t *ArgNames, void **ArgMappers) { TIMESCOPE_WITH_IDENT(Loc); + /// RAII to establish tool anchors before and after data end + OmptInterfaceTargetRAII TargetDataEndRAII(DeviceId, ompt_target_exit_data); targetDataMapper(Loc, DeviceId, ArgNum, ArgsBase, Args, ArgSizes, ArgTypes, ArgNames, ArgMappers, targetDataEnd, "Exiting OpenMP data region", "end"); @@ -172,6 +237,8 @@ map_var_info_t *ArgNames, void **ArgMappers) { TIMESCOPE_WITH_IDENT(Loc); + /// RAII to establish tool anchors before and after data update + OmptInterfaceTargetRAII TargetDataUpdateRAII(DeviceId, ompt_target_update); targetDataMapper( Loc, DeviceId, ArgNum, ArgsBase, Args, ArgSizes, ArgTypes, ArgNames, ArgMappers, targetDataUpdate, "Updating OpenMP data", "update"); @@ -270,6 +337,8 @@ DeviceTy &Device = *PM->Devices[DeviceId]; TargetAsyncInfoTy TargetAsyncInfo(Device); + /// RAII to establish tool anchors before and after target region + OmptInterfaceTargetRAII TargetRAII(DeviceId, ompt_target); AsyncInfoTy &AsyncInfo = TargetAsyncInfo; int Rc = OFFLOAD_SUCCESS; @@ -336,7 +405,8 @@ return OMP_TGT_FAIL; } DeviceTy &Device = *PM->Devices[DeviceId]; - + /// RAII to establish tool anchors before and after target region + OmptInterfaceTargetRAII TargetRAII(DeviceId, ompt_target); AsyncInfoTy AsyncInfo(Device); int Rc = target_replay(Loc, Device, HostPtr, DeviceMemory, DeviceMemorySize, TgtArgs, TgtOffsets, NumArgs, NumTeams, ThreadLimit, diff --git a/openmp/libomptarget/src/omptarget.cpp b/openmp/libomptarget/src/omptarget.cpp --- a/openmp/libomptarget/src/omptarget.cpp +++ b/openmp/libomptarget/src/omptarget.cpp @@ -12,6 +12,8 @@ //===----------------------------------------------------------------------===// #include "omptarget.h" +#include "OmptCallback.h" +#include "OmptInterface.h" #include "device.h" #include "private.h" #include "rtl.h" @@ -24,6 +26,26 @@ #include using llvm::SmallVector; +using namespace llvm::omp::target::ompt; + +/// RAII used to invoke callbacks before and after kernel launch +struct OmptInterfaceTargetSubmitRAII { + OmptInterfaceTargetSubmitRAII(int32_t Teams) : NumTeams{Teams} { + performIfOmptInitialized(beginSubmit()); + } + ~OmptInterfaceTargetSubmitRAII() { performIfOmptInitialized(endSubmit()); } + +private: + int32_t NumTeams; // Number of teams + void beginSubmit() { + // if (!OmptEnabled) return; + RegionInterface.beginTargetSubmit(NumTeams); + } + void endSubmit() { + // if (!OmptEnabled) return; + RegionInterface.endTargetSubmit(NumTeams); + } +}; int AsyncInfoTy::synchronize() { int Result = OFFLOAD_SUCCESS; @@ -1670,6 +1692,15 @@ { assert(KernelArgs.NumArgs == TgtArgs.size() && "Argument count mismatch!"); TIMESCOPE_WITH_NAME_AND_IDENT("Initiate Kernel Launch", Loc); + +#ifdef OMPT_SUPPORT + assert(KernelArgs.NumTeams[1] == 0 && KernelArgs.NumTeams[2] == 0 && + "Multi dimensional launch not supported yet."); + /// RAII to establish tool anchors before and after kernel launch + int32_t NumTeams = KernelArgs.NumTeams[0]; + OmptInterfaceTargetSubmitRAII TargetSubmitRAII(NumTeams); +#endif + Ret = Device.launchKernel(TgtEntryPtr, TgtArgs.data(), TgtOffsets.data(), KernelArgs, AsyncInfo); } diff --git a/openmp/libomptarget/test/ompt/veccopy.c b/openmp/libomptarget/test/ompt/veccopy.c --- a/openmp/libomptarget/test/ompt/veccopy.c +++ b/openmp/libomptarget/test/ompt/veccopy.c @@ -55,10 +55,29 @@ return rc; } -/// CHECK: Could not register callback 'ompt_callback_target_data_op' -/// CHECK: Could not register callback 'ompt_callback_target' -/// CHECK: Could not register callback 'ompt_callback_target_submit' - /// CHECK: Callback Init: /// CHECK: Callback Load: +/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2 + +/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2 /// CHECK: Callback Fini: diff --git a/openmp/libomptarget/test/ompt/veccopy_disallow_both.c b/openmp/libomptarget/test/ompt/veccopy_disallow_both.c --- a/openmp/libomptarget/test/ompt/veccopy_disallow_both.c +++ b/openmp/libomptarget/test/ompt/veccopy_disallow_both.c @@ -58,10 +58,44 @@ return rc; } -/// CHECK: Could not register callback 'ompt_callback_target_data_op' -/// CHECK: Could not register callback 'ompt_callback_target' -/// CHECK: Could not register callback 'ompt_callback_target_submit' - /// CHECK: Callback Init: /// CHECK: Callback Load: +/// CHECK: Callback Target EMI: kind=1 endpoint=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback Target EMI: kind=1 endpoint=2 +/// CHECK: Callback Target EMI: kind=1 endpoint=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback Target EMI: kind=1 endpoint=2 /// CHECK: Callback Fini: diff --git a/openmp/libomptarget/test/ompt/veccopy_emi.c b/openmp/libomptarget/test/ompt/veccopy_emi.c --- a/openmp/libomptarget/test/ompt/veccopy_emi.c +++ b/openmp/libomptarget/test/ompt/veccopy_emi.c @@ -56,10 +56,46 @@ return rc; } -/// CHECK: Could not register callback 'ompt_callback_target_data_op_emi' -/// CHECK: Could not register callback 'ompt_callback_target_emi' -/// CHECK: Could not register callback 'ompt_callback_target_submit_emi' - /// CHECK: Callback Init: /// CHECK: Callback Load: +/// CHECK: Callback Target EMI: kind=1 endpoint=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback Submit EMI: endpoint=1 req_num_teams=1 +/// CHECK: Callback Submit EMI: endpoint=2 req_num_teams=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback Target EMI: kind=1 endpoint=2 +/// CHECK: Callback Target EMI: kind=1 endpoint=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback Submit EMI: endpoint=1 req_num_teams=0 +/// CHECK: Callback Submit EMI: endpoint=2 req_num_teams=0 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback Target EMI: kind=1 endpoint=2 /// CHECK: Callback Fini: diff --git a/openmp/libomptarget/test/ompt/veccopy_emi_map.c b/openmp/libomptarget/test/ompt/veccopy_emi_map.c --- a/openmp/libomptarget/test/ompt/veccopy_emi_map.c +++ b/openmp/libomptarget/test/ompt/veccopy_emi_map.c @@ -56,11 +56,47 @@ return rc; } -/// CHECK: Could not register callback 'ompt_callback_target_data_op_emi' -/// CHECK: Could not register callback 'ompt_callback_target_emi' -/// CHECK: Could not register callback 'ompt_callback_target_submit_emi' -/// CHECK: Could not register callback 'ompt_callback_target_map_emi' - +/// CHECK: 0: Could not register callback 'ompt_callback_target_map_emi' /// CHECK: Callback Init: /// CHECK: Callback Load: +/// CHECK: Callback Target EMI: kind=1 endpoint=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback Submit EMI: endpoint=1 req_num_teams=1 +/// CHECK: Callback Submit EMI: endpoint=2 req_num_teams=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback Target EMI: kind=1 endpoint=2 +/// CHECK: Callback Target EMI: kind=1 endpoint=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback Submit EMI: endpoint=1 req_num_teams=0 +/// CHECK: Callback Submit EMI: endpoint=2 req_num_teams=0 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback Target EMI: kind=1 endpoint=2 /// CHECK: Callback Fini: diff --git a/openmp/libomptarget/test/ompt/veccopy_map.c b/openmp/libomptarget/test/ompt/veccopy_map.c --- a/openmp/libomptarget/test/ompt/veccopy_map.c +++ b/openmp/libomptarget/test/ompt/veccopy_map.c @@ -55,10 +55,31 @@ return rc; } -/// CHECK: Could not register callback 'ompt_callback_target_data_op' -/// CHECK: Could not register callback 'ompt_callback_target' -/// CHECK: Could not register callback 'ompt_callback_target_submit' +/// CHECK: 0: Could not register callback 'ompt_callback_target_map' /// CHECK: Callback Init: /// CHECK: Callback Load: +/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2 + +/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2 /// CHECK: Callback Fini: diff --git a/openmp/libomptarget/test/ompt/veccopy_no_device_init.c b/openmp/libomptarget/test/ompt/veccopy_no_device_init.c --- a/openmp/libomptarget/test/ompt/veccopy_no_device_init.c +++ b/openmp/libomptarget/test/ompt/veccopy_no_device_init.c @@ -54,10 +54,29 @@ return rc; } -/// CHECK: Could not register callback 'ompt_callback_target_data_op' -/// CHECK: Could not register callback 'ompt_callback_target' -/// CHECK: Could not register callback 'ompt_callback_target_submit' - /// CHECK-NOT: Callback Init: /// CHECK-NOT: Callback Load: +/// CHECK-NOT: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK-NOT: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK-NOT: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2 + +/// CHECK-NOT: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK-NOT: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK-NOT: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2 /// CHECK-NOT: Callback Fini: diff --git a/openmp/libomptarget/test/ompt/veccopy_wrong_return.c b/openmp/libomptarget/test/ompt/veccopy_wrong_return.c --- a/openmp/libomptarget/test/ompt/veccopy_wrong_return.c +++ b/openmp/libomptarget/test/ompt/veccopy_wrong_return.c @@ -54,10 +54,29 @@ return rc; } -/// CHECK: Could not register callback 'ompt_callback_target_data_op' -/// CHECK: Could not register callback 'ompt_callback_target' -/// CHECK: Could not register callback 'ompt_callback_target_submit' - /// CHECK-NOT: Callback Init: /// CHECK-NOT: Callback Load: -/// CHECK-NOT: Callback Fini: +/// CHECK-NOT: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK-NOT: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK-NOT: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2 + +/// CHECK-NOT: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK-NOT: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK-NOT: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2 +/// CHECK-NOT: Callback Fini diff --git a/openmp/runtime/src/ompt-event-specific.h b/openmp/runtime/src/ompt-event-specific.h --- a/openmp/runtime/src/ompt-event-specific.h +++ b/openmp/runtime/src/ompt-event-specific.h @@ -55,13 +55,12 @@ #define ompt_callback_implicit_task_implemented ompt_event_MAY_ALWAYS -#define ompt_callback_target_implemented ompt_event_UNIMPLEMENTED -#define ompt_callback_target_emi_implemented ompt_event_UNIMPLEMENTED -#define ompt_callback_target_data_op_implemented ompt_event_UNIMPLEMENTED -#define ompt_callback_target_data_op_emi_implemented ompt_event_UNIMPLEMENTED -#define ompt_callback_target_submit_implemented ompt_event_UNIMPLEMENTED -#define ompt_callback_target_submit_emi_implemented ompt_event_UNIMPLEMENTED - +#define ompt_callback_target_implemented ompt_event_MAY_ALWAYS +#define ompt_callback_target_emi_implemented ompt_event_MAY_ALWAYS +#define ompt_callback_target_data_op_implemented ompt_event_MAY_ALWAYS +#define ompt_callback_target_data_op_emi_implemented ompt_event_MAY_ALWAYS +#define ompt_callback_target_submit_implemented ompt_event_MAY_ALWAYS +#define ompt_callback_target_submit_emi_implemented ompt_event_MAY_ALWAYS #define ompt_callback_control_tool_implemented ompt_event_MAY_ALWAYS #define ompt_callback_device_initialize_implemented ompt_event_MAY_ALWAYS