[OpenMP] Disabled profiling in `libomp` by default to unblock link errors

[tianshilei1992]

Summary

Link error occurred when time profiling in libomp is enabled by default
because libomp is assumed to be a C library but the dependence on
libLLVMSupport for profiling is a C++ library. Currently the issue blocks all
OpenMP tests in Phabricator.

This patch set a new CMake option OPENMP_ENABLE_LIBOMP_PROFILING to
enable/disable the feature. By default it is disabled. Note that once time
profiling is enabled for libomp, it becomes a C++ library.

[tianshilei1992]

added doc

[jdoerfert]

LG, we need to backport this too. And we need to document that profiling multi-threaded applications needs this flag in the build.

openmp/docs/design/Runtimes.rst

1

changes in this file seem unrelated.

[jdoerfert]

LG, thanks

openmp/docs/design/Runtimes.rst

54

to -> into

[jdoerfert]

wrong selection :(

[tianshilei1992]

comments

[mstorsjo]

The cherrypick of this commit to the 12.x branch broke building of openmp there, with errors like these:

CMake Error at CMakeLists.txt:93 (add_subdirectory):
  The binary directory

    /build/llvm-project/openmp/build-i686/runtime

  is already used to build a source directory.  It cannot be used to build
  source directory

    /build/llvm-project/openmp/runtime

  Specify a unique binary directory name.

That's because on the main branch, D95398 moved the add_subdirectory(runtime), but that bit isn't cherrypicked to the release branch, and the cherry pick conflict seems to have been resolved incorrectly.

[tianshilei1992]

In D95585#2541372, @mstorsjo wrote:

The cherrypick of this commit to the 12.x branch broke building of openmp there, with errors like these:

CMake Error at CMakeLists.txt:93 (add_subdirectory):
  The binary directory

    /build/llvm-project/openmp/build-i686/runtime

  is already used to build a source directory.  It cannot be used to build
  source directory

    /build/llvm-project/openmp/runtime

  Specify a unique binary directory name.

That's because on the main branch, D95398 moved the add_subdirectory(runtime), but that bit isn't cherrypicked to the release branch, and the cherry pick conflict seems to have been resolved incorrectly.

Thanks for the report. Now it is fixed.

openmp/CMakeLists.txt

if (NOT LIBOMPTARGET_LLVM_INCLUDE_DIRS)
   message(STATUS "Missing definition for LIBOMPTARGET_LLVM_INCLUDE_DIRS, disabling libomptarget")
   set(ENABLE_LIBOMPTARGET OFF)
 endif()
 
 option(OPENMP_ENABLE_LIBOMPTARGET "Enable building libomptarget for offloading."
        ${ENABLE_LIBOMPTARGET})
 option(OPENMP_ENABLE_LIBOMPTARGET_PROFILING "Enable time profiling for libomptarget."
        ${ENABLE_LIBOMPTARGET})
+option(OPENMP_ENABLE_LIBOMP_PROFILING "Enable time profiling for libomp." OFF)
 
 # Build host runtime library, after LIBOMPTARGET variables are set since they are needed
 # to enable time profiling support in the OpenMP runtime.
 add_subdirectory(runtime)
 
 if (OPENMP_ENABLE_LIBOMPTARGET)
   # Check that the library can actually be built.
   if (APPLE OR WIN32)

openmp/docs/design/Runtimes.rst

 .. _openmp_runtimes:
 
 LLVM/OpenMP Runtimes
 ====================
 
 There are four distinct types of LLVM/OpenMP runtimes 
 
 LLVM/OpenMP Host Runtime (``libomp``)
 -------------------------------------

 LIBOMPTARGET_PROFILE
 """"""""""""""""""""
 ``LIBOMPTARGET_PROFILE`` allows ``libomptarget`` to generate time profile output
 similar to Clang's ``-ftime-trace`` option. This generates a JSON file based on
 `Chrome Tracing`_ that can be viewed with ``chrome://tracing`` or the
 `Speedscope App`_. Building this feature depends on the `LLVM Support Library`_
 for time trace output. Using this library is enabled by default when building
 using the CMake option ``OPENMP_ENABLE_LIBOMPTARGET_PROFILING``. The output will
-be saved to the filename specified by the environment variable.
+be saved to the filename specified by the environment variable. For multi-threaded
+applications, profiling in ``libomp`` is also needed. Setting the CMake option
+``OPENMP_ENABLE_LIBOMP_PROFILING=ON`` to enable the feature. Note that this will
+turn ``libomp`` into a C++ library.
 
 .. _`Chrome Tracing`: https://www.chromium.org/developers/how-tos/trace-event-profiling-tool
 
 .. _`Speedscope App`: https://www.speedscope.app/
 
 .. _`LLVM Support Library`: https://llvm.org/docs/SupportLibrary.html
 
 LIBOMPTARGET_MEMORY_MANAGER_THRESHOLD

openmp/runtime/CMakeLists.txt

 if(${OPENMP_STANDALONE_BUILD})
   # If adding a new architecture, take a look at cmake/LibompGetArchitecture.cmake
   libomp_get_architecture(LIBOMP_DETECTED_ARCH)
   set(LIBOMP_ARCH ${LIBOMP_DETECTED_ARCH} CACHE STRING
     "The architecture to build for (x86_64/i386/arm/ppc64/ppc64le/aarch64/mic/mips/mips64/riscv64).")
   # Should assertions be enabled?  They are on by default.
   set(LIBOMP_ENABLE_ASSERTIONS TRUE CACHE BOOL
     "enable assertions?")
-  set(LIBOMPTARGET_PROFILING_SUPPORT FALSE)
 else() # Part of LLVM build
   # Determine the native architecture from LLVM.
   string(TOLOWER "${LLVM_TARGET_ARCH}" LIBOMP_NATIVE_ARCH)
   if( LIBOMP_NATIVE_ARCH STREQUAL "host" )
     string(REGEX MATCH "^[^-]*" LIBOMP_NATIVE_ARCH ${LLVM_HOST_TRIPLE})
   endif ()
   if(LIBOMP_NATIVE_ARCH MATCHES "i[2-6]86")
     set(LIBOMP_ARCH i386)
elseif(LIBOMP_NATIVE_ARCH MATCHES "arm")
     set(LIBOMP_ARCH arm)
   elseif(LIBOMP_NATIVE_ARCH MATCHES "riscv64")
     set(LIBOMP_ARCH riscv64)
   else()
     # last ditch effort
     libomp_get_architecture(LIBOMP_ARCH)
   endif ()
   set(LIBOMP_ENABLE_ASSERTIONS ${LLVM_ENABLE_ASSERTIONS})
-  # Time profiling support
-  set(LIBOMPTARGET_PROFILING_SUPPORT ${OPENMP_ENABLE_LIBOMPTARGET_PROFILING})
 endif()
 
+# Time profiling support
+set(LIBOMP_PROFILING_SUPPORT ${OPENMP_ENABLE_LIBOMP_PROFILING})
+
 # FUJITSU A64FX is a special processor because its cache line size is 256.
 # We need to pass this information into kmp_config.h.
 if(LIBOMP_ARCH STREQUAL "aarch64")
   libomp_is_aarch64_a64fx(LIBOMP_DETECT_AARCH64_A64FX)
   if (LIBOMP_DETECT_AARCH64_A64FX)
     set(LIBOMP_ARCH "aarch64_a64fx")
     set(LIBOMP_ARCH_AARCH64_A64FX TRUE)
   endif()

openmp/runtime/src/CMakeLists.txt

include_directories(
   ${LIBOMP_SRC_DIR}/i18n
   ${LIBOMP_INC_DIR}
   ${LIBOMP_SRC_DIR}/thirdparty/ittnotify
 )
 if(${LIBOMP_USE_HWLOC})
   include_directories(${LIBOMP_HWLOC_INSTALL_DIR}/include)
 endif()
 
-# Building with time profiling support for libomptarget requires
-# LLVM directory includes.
-if(LIBOMPTARGET_PROFILING_SUPPORT)
+# Building with time profiling support requires LLVM directory includes.
+if(LIBOMP_PROFILING_SUPPORT)
   include_directories(
     ${LLVM_MAIN_INCLUDE_DIR}
     ${LLVM_INCLUDE_DIR}
   )
 endif()
 
 # Getting correct source files to build library
 set(LIBOMP_CXXFILES)
else()
   set(LIBOMP_LINKER_LANGUAGE CXX)
 endif()
 
 # Add the OpenMP library
 libomp_get_ldflags(LIBOMP_CONFIGURED_LDFLAGS)
 
 libomp_get_libflags(LIBOMP_CONFIGURED_LIBFLAGS)
 # Build libomp library. Add LLVMSupport dependency if building in-tree with libomptarget profiling enabled.
-if(OPENMP_STANDALONE_BUILD OR (NOT OPENMP_ENABLE_LIBOMPTARGET_PROFILING))
+if(OPENMP_STANDALONE_BUILD OR (NOT OPENMP_ENABLE_LIBOMP_PROFILING))
   add_library(omp ${LIBOMP_LIBRARY_KIND} ${LIBOMP_SOURCE_FILES})
   # Linking command will include libraries in LIBOMP_CONFIGURED_LIBFLAGS
   target_link_libraries(omp ${LIBOMP_CONFIGURED_LIBFLAGS} ${CMAKE_DL_LIBS})
 else()
   add_llvm_library(omp ${LIBOMP_LIBRARY_KIND} ${LIBOMP_SOURCE_FILES} PARTIAL_SOURCES_INTENDED
     LINK_LIBS ${LIBOMP_CONFIGURED_LIBFLAGS} ${CMAKE_DL_LIBS}
     LINK_COMPONENTS Support
     )
+  # libomp must be a C++ library such that it can link libLLVMSupport
+  set(LIBOMP_LINKER_LANGUAGE CXX)
 endif()
 
 set_target_properties(omp PROPERTIES
   PREFIX "" SUFFIX "" OUTPUT_NAME "${LIBOMP_LIB_FILE}"
   LINK_FLAGS "${LIBOMP_CONFIGURED_LDFLAGS}"
   LINKER_LANGUAGE ${LIBOMP_LINKER_LANGUAGE}
 )
 

openmp/runtime/src/kmp_config.h.cmake

 #cmakedefine01 LIBOMP_HAVE___RDTSC
 #define KMP_HAVE___RDTSC LIBOMP_HAVE___RDTSC
 #cmakedefine01 LIBOMP_USE_DEBUGGER
 #define USE_DEBUGGER LIBOMP_USE_DEBUGGER
 #cmakedefine01 LIBOMP_OMPT_DEBUG
 #define OMPT_DEBUG LIBOMP_OMPT_DEBUG
 #cmakedefine01 LIBOMP_OMPT_SUPPORT
 #define OMPT_SUPPORT LIBOMP_OMPT_SUPPORT
-#cmakedefine01 LIBOMPTARGET_PROFILING_SUPPORT
-#define OMPTARGET_PROFILING_SUPPORT LIBOMPTARGET_PROFILING_SUPPORT
+#cmakedefine01 LIBOMP_PROFILING_SUPPORT
+#define OMP_PROFILING_SUPPORT LIBOMP_PROFILING_SUPPORT
 #cmakedefine01 LIBOMP_OMPT_OPTIONAL
 #define OMPT_OPTIONAL LIBOMP_OMPT_OPTIONAL
 #cmakedefine01 LIBOMP_USE_ADAPTIVE_LOCKS
 #define KMP_USE_ADAPTIVE_LOCKS LIBOMP_USE_ADAPTIVE_LOCKS
 #define KMP_DEBUG_ADAPTIVE_LOCKS 0
 #cmakedefine01 LIBOMP_USE_INTERNODE_ALIGNMENT
 #define KMP_USE_INTERNODE_ALIGNMENT LIBOMP_USE_INTERNODE_ALIGNMENT
 #cmakedefine01 LIBOMP_ENABLE_ASSERTIONS

openmp/runtime/src/kmp_runtime.cpp

 #if KMP_USE_HIER_SCHED
 #include "kmp_dispatch_hier.h"
 #endif
 
 #if OMPT_SUPPORT
 #include "ompt-specific.h"
 #endif
 
-#if OMPTARGET_PROFILING_SUPPORT
+#if OMP_PROFILING_SUPPORT
 #include "llvm/Support/TimeProfiler.h"
 static char *ProfileTraceFile = nullptr;
 #endif
 
 /* these are temporary issues to be dealt with */
 #define KMP_USE_PRCTL 0
 
 #if KMP_OS_WINDOWS

 #endif /* KMP_ADJUST_BLOCKTIME */
 
   KMP_MB();
 }
 
 /* ------------------------------------------------------------------------ */
 
 void *__kmp_launch_thread(kmp_info_t *this_thr) {
-#if OMPTARGET_PROFILING_SUPPORT
+#if OMP_PROFILING_SUPPORT
   ProfileTraceFile = getenv("LIBOMPTARGET_PROFILE");
   // TODO: add a configuration option for time granularity
   if (ProfileTraceFile)
     llvm::timeTraceProfilerInitialize(500 /* us */, "libomptarget");
 #endif
 
   int gtid = this_thr->th.th_info.ds.ds_gtid;
   /*    void                 *stack_data;*/
#endif
 
   this_thr->th.th_task_team = NULL;
   /* run the destructors for the threadprivate data for this thread */
   __kmp_common_destroy_gtid(gtid);
 
   KA_TRACE(10, ("__kmp_launch_thread: T#%d done\n", gtid));
   KMP_MB();
 
-#if OMPTARGET_PROFILING_SUPPORT
+#if OMP_PROFILING_SUPPORT
   llvm::timeTraceProfilerFinishThread();
 #endif
   return this_thr;
 }
 
 /* ------------------------------------------------------------------------ */
 
 void __kmp_internal_end_dest(void *specific_gtid) {