Make LIBOMP_USE_ITT_NOTIFY a setting that can be enabled or disabled

[jmellorcrummey]

Summary

Make LIBOMP_USE_ITT_NOTIFY a setting that can be enabled or disabled.

On Blue Gene/Q, having LIBOMP_USE_ITT_NOTIFY support compiled into a statically-linked binary causes a failure at runtime because dlopen fails.

This patch changes LIBOMP_USE_ITT_NOTIFY to a cacheable configuration setting that can be disabled.

[jmellorcrummey]

Remove comment that described ITT as always on.

[jlpeyton]

Adding Olga and Andrey

Olga, Andrey,

Why are there two macros used throughout the codebase, USE_ITT_NOTIFY and USE_ITT_BUILD? Do they have different meanings? The LIBOMP_USE_ITT_NOTIFY (from this patch) is translated to USE_ITT_NOTIFY, but USE_ITT_BUILD is hard set to 1 in kmp_config.h.cmake. I guess I just need clarifications on them.

[AndreyChurbanov]

The USE_ITT_BUILD=0 removes all the ITT code from the sources, while the USE_ITT_NOTIFY=0 with USE_ITT_BUILD=1 leaves all ITT calls as empty routines. This is old code, and I guess two macros were introduced long ago for better debugging of the ittnotify interfaces. I doubt we need empty routines nowadays, so we can think of leaving only one macro that controls whether we have fully functional ittnotify or nothing. This should be enough. Whether it is called USE_ITT_NOTIFY or USE_ITT_BUILD does not make big difference to me, probably USE_ITT_NOTIFY better corresponds with ittnotify library name.

[jmellorcrummey]

Now that I have heard some thoughts from Intel, this patch does the following:

(1) improve the patch to replace hardwired "#define USE_ITT_BUILD 1" with #define USE_ITT_BUILD LIBOMP_ITT_NOTIFY" so that when LIBOMP_ITT_NOTIFY is turned off in cmake, USE_ITT_BUILD is set to 0, which causes all ITT support to be omitted.
(2) fix code that broke when USE_ITT_BUILD was turned off.

[jlpeyton]

LGTM

openmp/trunk/runtime/CMakeLists.txt

 elseif("${libomp_build_type_lowercase}" STREQUAL "debug")
   set(DEBUG_BUILD TRUE)
 elseif("${libomp_build_type_lowercase}" STREQUAL "relwithdebinfo")
   set(RELWITHDEBINFO_BUILD TRUE)
 elseif("${libomp_build_type_lowercase}" STREQUAL "minsizerel")
   set(MINSIZEREL_BUILD TRUE)
 endif()
 
-# Include itt notify interface? Right now, always.
-set(LIBOMP_USE_ITT_NOTIFY TRUE)
+# Include itt notify interface?
+set(LIBOMP_USE_ITT_NOTIFY TRUE CACHE BOOL
+  "Enable ITT notify?")
 
 # normal, profile, stubs library.
 set(NORMAL_LIBRARY FALSE)
 set(STUBS_LIBRARY FALSE)
 set(PROFILE_LIBRARY FALSE)
 if("${LIBOMP_LIB_TYPE}" STREQUAL "normal")
   set(NORMAL_LIBRARY TRUE)
 elseif("${LIBOMP_LIB_TYPE}" STREQUAL "profile")
if(${LIBOMP_STANDALONE_BUILD})
   if(${LIBOMP_VERSION_BUILD} STREQUAL 00000000)
     set(LIBOMP_BUILD Development)
   else()
     set(LIBOMP_BUILD ${LIBOMP_VERSION_BUILD})
   endif()
   libomp_say("Build                -- ${LIBOMP_BUILD}")
   libomp_say("Use Stats-gathering  -- ${LIBOMP_STATS}")
   libomp_say("Use Debugger-support -- ${LIBOMP_USE_DEBUGGER}")
+  libomp_say("Use ITT notify       -- ${LIBOMP_USE_ITT_NOTIFY}")
   libomp_say("Use OMPT-support     -- ${LIBOMP_OMPT_SUPPORT}")
   if(${LIBOMP_OMPT_SUPPORT})
     libomp_say("Use OMPT-blame       -- ${LIBOMP_OMPT_BLAME}")
     libomp_say("Use OMPT-trace       -- ${LIBOMP_OMPT_TRACE}")
   endif()
   libomp_say("Use Adaptive locks   -- ${LIBOMP_USE_ADAPTIVE_LOCKS}")
   libomp_say("Use quad precision   -- ${LIBOMP_USE_QUAD_PRECISION}")
   libomp_say("Use Hwloc library    -- ${LIBOMP_USE_HWLOC}")
 endif()
 
 add_subdirectory(src)
 add_subdirectory(test)
 

openmp/trunk/runtime/src/kmp_config.h.cmake

 # define BUILD_I8 1
 #endif
 
 #define KMP_DYNAMIC_LIB 1
 #define KMP_NESTED_HOT_TEAMS 1
 #define KMP_ADJUST_BLOCKTIME 1
 #define BUILD_PARALLEL_ORDERED 1
 #define KMP_ASM_INTRINS 1
-#define USE_ITT_BUILD 1
+#define USE_ITT_BUILD LIBOMP_USE_ITT_NOTIFY
 #define INTEL_ITTNOTIFY_PREFIX __kmp_itt_
 #if ! KMP_MIC
 # define USE_LOAD_BALANCE 1
 #endif
 #if ! (KMP_OS_WINDOWS || KMP_OS_DARWIN)
 # define KMP_TDATA_GTID 1
 #endif
 #if STUBS_LIBRARY

openmp/trunk/runtime/src/kmp_csupport.c

__kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
     this_thr    = __kmp_threads[ global_tid ];
     serial_team = this_thr->th.th_serial_team;
 
    #if OMP_41_ENABLED
    kmp_task_team_t *   task_team = this_thr->th.th_task_team;
 
    // we need to wait for the proxy tasks before finishing the thread
    if ( task_team != NULL && task_team->tt.tt_found_proxy_tasks )
-        __kmp_task_team_wait(this_thr, serial_team, NULL ); // is an ITT object needed here?
+        __kmp_task_team_wait(this_thr, serial_team USE_ITT_BUILD_ARG(NULL) ); // is an ITT object needed here?
    #endif
 
     KMP_MB();
     KMP_DEBUG_ASSERT( serial_team );
     KMP_ASSERT(       serial_team -> t.t_serialized );
     KMP_DEBUG_ASSERT( this_thr -> th.th_team == serial_team );
     KMP_DEBUG_ASSERT( serial_team != this_thr->th.th_root->r.r_root_team );
     KMP_DEBUG_ASSERT( serial_team -> t.t_threads );

openmp/trunk/runtime/src/kmp_runtime.c

#endif
 
     /* initialize the thread, get it ready to go */
     __kmp_initialize_info( root_thread, root->r.r_root_team, 0, gtid );
     TCW_4(__kmp_init_gtid, TRUE);
 
     /* prepare the master thread for get_gtid() */
     __kmp_gtid_set_specific( gtid );
 
+#if USE_ITT_BUILD
     __kmp_itt_thread_name( gtid );
+#endif /* USE_ITT_BUILD */
 
     #ifdef KMP_TDATA_GTID
         __kmp_gtid = gtid;
     #endif
     __kmp_create_worker( gtid, root_thread, __kmp_stksize );
     KMP_DEBUG_ASSERT( __kmp_gtid_get_specific() == gtid );
 
     KA_TRACE( 20, ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, plain=%u\n",
#if OMP_41_ENABLED
    kmp_task_team_t *   task_team = thread->th.th_task_team;
 
    // we need to wait for the proxy tasks before finishing the thread
    if ( task_team != NULL && task_team->tt.tt_found_proxy_tasks ) {
 #if OMPT_SUPPORT
         // the runtime is shutting down so we won't report any events
         thread->th.ompt_thread_info.state = ompt_state_undefined;
 #endif
-        __kmp_task_team_wait(thread, team, NULL );
+        __kmp_task_team_wait(thread, team USE_ITT_BUILD_ARG(NULL));
    }
 #endif
 
     __kmp_reset_root(gtid, root);
 
     /* free up this thread slot */
     __kmp_gtid_set_specific( KMP_GTID_DNE );
 #ifdef KMP_TDATA_GTID