[OpenMP] Add time profiling support in libomp

[ggeorgakoudis]

Summary

Profiling has been recently implemented in libomptarget (D93055). This patch enables time profiling support for libomptarget in libomp, to support profiling of multi-threaded execution of offloaded regions.

[ggeorgakoudis]

Fix setting LIBOMPTARGET_PROFILING_SUPPORT

[jdoerfert]

LGTM

[JonChesterfield]

This breaks the build for me. Compiling openmp via ENABLE_RUNTIMES without any extra options, the include path isn't set up so kmp_runtime.cpp fails to find the header.

[ggeorgakoudis]

In D94855#2513249, @JonChesterfield wrote:

This breaks the build for me. Compiling openmp via ENABLE_RUNTIMES without any extra options, the include path isn't set up so kmp_runtime.cpp fails to find the header.

Thanks for spotting that @JonChesterfield. Will fix soon.

[tianshilei1992]

I see two issues in this patch. LIBOMPTARGET_LLVM_INCLUDE_DIRS is not included when building libomp.so therefore the header will not be found (in my case). Another probably bigger one is D95371. From my observation, code in runtime is very conservative in terms of using C++.

[tianshilei1992]

We need to revise the decision in this patch. Per @AndreyChurbanov in D95371, libomp.so is intended to be a C library. Linking a C library with a C++ static library doesn't work, as shown in D95371.

openmp/runtime/CMakeLists.txt

 if(${OPENMP_STANDALONE_BUILD})
   # If adding a new architecture, take a look at cmake/LibompGetArchitecture.cmake
   libomp_get_architecture(LIBOMP_DETECTED_ARCH)
   set(LIBOMP_ARCH ${LIBOMP_DETECTED_ARCH} CACHE STRING
     "The architecture to build for (x86_64/i386/arm/ppc64/ppc64le/aarch64/mic/mips/mips64/riscv64).")
   # Should assertions be enabled?  They are on by default.
   set(LIBOMP_ENABLE_ASSERTIONS TRUE CACHE BOOL
     "enable assertions?")
+  set(LIBOMPTARGET_PROFILING_SUPPORT FALSE)
 else() # Part of LLVM build
   # Determine the native architecture from LLVM.
   string(TOLOWER "${LLVM_TARGET_ARCH}" LIBOMP_NATIVE_ARCH)
   if( LIBOMP_NATIVE_ARCH STREQUAL "host" )
     string(REGEX MATCH "^[^-]*" LIBOMP_NATIVE_ARCH ${LLVM_HOST_TRIPLE})
   endif ()
   if(LIBOMP_NATIVE_ARCH MATCHES "i[2-6]86")
     set(LIBOMP_ARCH i386)
elseif(LIBOMP_NATIVE_ARCH MATCHES "arm")
     set(LIBOMP_ARCH arm)
   elseif(LIBOMP_NATIVE_ARCH MATCHES "riscv64")
     set(LIBOMP_ARCH riscv64)
   else()
     # last ditch effort
     libomp_get_architecture(LIBOMP_ARCH)
   endif ()
   set(LIBOMP_ENABLE_ASSERTIONS ${LLVM_ENABLE_ASSERTIONS})
+  # Time profiling support
+  set(LIBOMPTARGET_PROFILING_SUPPORT ${OPENMP_ENABLE_LIBOMPTARGET_PROFILING})
 endif()
 
 # FUJITSU A64FX is a special processor because its cache line size is 256.
 # We need to pass this information into kmp_config.h.
 if(LIBOMP_ARCH STREQUAL "aarch64")
   libomp_is_aarch64_a64fx(LIBOMP_DETECT_AARCH64_A64FX)
   if (LIBOMP_DETECT_AARCH64_A64FX)
     set(LIBOMP_ARCH "aarch64_a64fx")

openmp/runtime/src/CMakeLists.txt

if(NOT ${LIBOMP_USE_STDCPPLIB})
   set(CMAKE_CXX_IMPLICIT_LINK_LIBRARIES)
 else()
   set(LIBOMP_LINKER_LANGUAGE CXX)
 endif()
 
 # Add the OpenMP library
 libomp_get_ldflags(LIBOMP_CONFIGURED_LDFLAGS)
 
+libomp_get_libflags(LIBOMP_CONFIGURED_LIBFLAGS)
+# Build libomp library. Add LLVMSupport dependency if building in-tree with libomptarget profiling enabled.
+if(OPENMP_STANDALONE_BUILD OR (NOT OPENMP_ENABLE_LIBOMPTARGET_PROFILING))
   add_library(omp ${LIBOMP_LIBRARY_KIND} ${LIBOMP_SOURCE_FILES})
+  # Linking command will include libraries in LIBOMP_CONFIGURED_LIBFLAGS
+  target_link_libraries(omp ${LIBOMP_CONFIGURED_LIBFLAGS} ${CMAKE_DL_LIBS})
+else()
+  add_llvm_library(omp ${LIBOMP_LIBRARY_KIND} ${LIBOMP_SOURCE_FILES} PARTIAL_SOURCES_INTENDED
+    LINK_LIBS ${LIBOMP_CONFIGURED_LIBFLAGS} ${CMAKE_DL_LIBS}
+    LINK_COMPONENTS Support
+    )
+endif()
 
 set_target_properties(omp PROPERTIES
   PREFIX "" SUFFIX "" OUTPUT_NAME "${LIBOMP_LIB_FILE}"
   LINK_FLAGS "${LIBOMP_CONFIGURED_LDFLAGS}"
   LINKER_LANGUAGE ${LIBOMP_LINKER_LANGUAGE}
 )
 
 # Get the library's location within the build tree for the unit tester
add_custom_command(TARGET omp POST_BUILD
     COMMAND ${CMAKE_COMMAND} -E create_symlink ${LIBOMP_LIB_FILE}
       libgomp${LIBOMP_LIBRARY_SUFFIX}
     COMMAND ${CMAKE_COMMAND} -E create_symlink ${LIBOMP_LIB_FILE}
       libiomp5${LIBOMP_LIBRARY_SUFFIX}
     WORKING_DIRECTORY ${LIBOMP_LIBRARY_DIR}
   )
 endif()
 
-# Linking command will include libraries in LIBOMP_CONFIGURED_LIBFLAGS
-libomp_get_libflags(LIBOMP_CONFIGURED_LIBFLAGS)
-target_link_libraries(omp ${LIBOMP_CONFIGURED_LIBFLAGS} ${CMAKE_DL_LIBS})
-
 # Create *.inc before compiling any sources
 # objects depend on : .inc files
 add_custom_target(libomp-needed-headers DEPENDS kmp_i18n_id.inc kmp_i18n_default.inc)
 add_dependencies(omp libomp-needed-headers)
 
 # Windows specific build rules
 if(WIN32)
   configure_file(libomp.rc.var libomp.rc @ONLY)

openmp/runtime/src/kmp_config.h.cmake

 #cmakedefine01 LIBOMP_HAVE___RDTSC
 #define KMP_HAVE___RDTSC LIBOMP_HAVE___RDTSC
 #cmakedefine01 LIBOMP_USE_DEBUGGER
 #define USE_DEBUGGER LIBOMP_USE_DEBUGGER
 #cmakedefine01 LIBOMP_OMPT_DEBUG
 #define OMPT_DEBUG LIBOMP_OMPT_DEBUG
 #cmakedefine01 LIBOMP_OMPT_SUPPORT
 #define OMPT_SUPPORT LIBOMP_OMPT_SUPPORT
+#cmakedefine01 LIBOMPTARGET_PROFILING_SUPPORT
+#define OMPTARGET_PROFILING_SUPPORT LIBOMPTARGET_PROFILING_SUPPORT
 #cmakedefine01 LIBOMP_OMPT_OPTIONAL
 #define OMPT_OPTIONAL LIBOMP_OMPT_OPTIONAL
 #cmakedefine01 LIBOMP_USE_ADAPTIVE_LOCKS
 #define KMP_USE_ADAPTIVE_LOCKS LIBOMP_USE_ADAPTIVE_LOCKS
 #define KMP_DEBUG_ADAPTIVE_LOCKS 0
 #cmakedefine01 LIBOMP_USE_INTERNODE_ALIGNMENT
 #define KMP_USE_INTERNODE_ALIGNMENT LIBOMP_USE_INTERNODE_ALIGNMENT
 #cmakedefine01 LIBOMP_ENABLE_ASSERTIONS

openmp/runtime/src/kmp_runtime.cpp

 #if KMP_USE_HIER_SCHED
 #include "kmp_dispatch_hier.h"
 #endif
 
 #if OMPT_SUPPORT
 #include "ompt-specific.h"
 #endif
 
+#if OMPTARGET_PROFILING_SUPPORT
+#include "llvm/Support/TimeProfiler.h"
+static char *ProfileTraceFile = nullptr;
+#endif
+
 /* these are temporary issues to be dealt with */
 #define KMP_USE_PRCTL 0
 
 #if KMP_OS_WINDOWS
 #include <process.h>
 #endif
 
 #include "tsan_annotations.h"

 #endif /* KMP_ADJUST_BLOCKTIME */
 
   KMP_MB();
 }
 
 /* ------------------------------------------------------------------------ */
 
 void *__kmp_launch_thread(kmp_info_t *this_thr) {
+#if OMPTARGET_PROFILING_SUPPORT
+  ProfileTraceFile = getenv("LIBOMPTARGET_PROFILE");
+  // TODO: add a configuration option for time granularity
+  if (ProfileTraceFile)
+    llvm::timeTraceProfilerInitialize(500 /* us */, "libomptarget");
+#endif
+
   int gtid = this_thr->th.th_info.ds.ds_gtid;
   /*    void                 *stack_data;*/
   kmp_team_t **volatile pteam;
 
   KMP_MB();
   KA_TRACE(10, ("__kmp_launch_thread: T#%d start\n", gtid));
 
   if (__kmp_env_consistency_check) {

 #endif
 
   this_thr->th.th_task_team = NULL;
   /* run the destructors for the threadprivate data for this thread */
   __kmp_common_destroy_gtid(gtid);
 
   KA_TRACE(10, ("__kmp_launch_thread: T#%d done\n", gtid));
   KMP_MB();
+
+#if OMPTARGET_PROFILING_SUPPORT
+  llvm::timeTraceProfilerFinishThread();
+#endif
   return this_thr;
 }
 
 /* ------------------------------------------------------------------------ */
 
 void __kmp_internal_end_dest(void *specific_gtid) {
   // Make sure no significant bits are lost
   int gtid;