Enable debugger support

[jlpeyton]

Summary

These changes enable external debuggers to conveniently interface with the LLVM OpenMP Library. Structures are added which describe the important internal structures of the OpenMP Library e.g., teams, threads, etc. This feature is turned on by default (LIBOMP_USE_DEBUGGER) and can be turned off with -DLIBOMP_USE_DEBUGGER=off.

[hfinkel]

runtime/src/kmp_debugger.h

24

Please document how, by whom, and when this can be set.

26

How is the ident parameter used? (please document that)

[jlpeyton]

On May 28th, 2015, Olga Malysheva wrote:

Hello!

Thank you for your comment. We'll add more comments for this code.

1. __kmp_debugging can be set by a debugger to notify OpenMP RTL that it is run under the debugger.

2. The ident parameter is used in __kmp_omp_num_threads() function to check if the number of threads was changed by the debugger for the parallel which location was specified by the ident.

Please, see usage of kmp_omp_num_threads() (together with kmp_debugging) in __kmp_fork_call() function.

Currently, Intel Debugger (IDB) is supported.

Regards,
Olga

Addressing Hal's requests with new patch.

Please document how, by whom, and when this can be set.
How is the ident parameter used? (please document that)

[hfinkel]

LGTM, thanks!

[jmellorcrummey]

While I don't object to this commit, I'll note that what we hope will be a standard for an OpenMP debugging interface known as OMPD is under active development. The current draft is posted here: https://github.com/OpenMPToolsInterface/OMPD-Technical-Report

A group at LLNL and Totalview is working to refine the spec. LLNL has developed patches to another version of the Intel runtime to support an OMPD prototype.

runtime/CMakeLists.txt

 set(LIBOMP_MICRO_TESTS false CACHE BOOL
     "Perform touch, relo, execstack, instr, and deps tests?")
 
 # - stats-gathering enables OpenMP stats where things like the number of 
 # parallel regions, clock ticks spent in particular openmp regions are recorded.
 set(LIBOMP_STATS false CACHE BOOL
     "Stats-Gathering functionality?")
 
+# - Code that allows the OpenMP library to conveniently interface with debuggers
+set(LIBOMP_USE_DEBUGGER true CACHE BOOL
+    "Enable debugger interface code?")
+
 # OMPT-support
 set(LIBOMP_OMPT_SUPPORT false CACHE BOOL
     "OMPT-support?")
 set(LIBOMP_OMPT_BLAME true CACHE BOOL
     "OMPT-blame?")
 set(LIBOMP_OMPT_TRACE true CACHE BOOL
     "OMPT-trace?")
 
if(${LIBOMP_STANDALONE_BUILD})
     # will say development if all zeros
     if("${build_number}" STREQUAL "00000000")
         set(build "development")
     else()
         set(build "${build_number}")
     endif()
     say("LIBOMP: Build                -- ${build}")
     say("LIBOMP: Stats-Gathering      -- ${LIBOMP_STATS}")
+    say("LIBOMP: Debugger-support     -- ${LIBOMP_USE_DEBUGGER}")
     say("LIBOMP: OMPT-support         -- ${LIBOMP_OMPT_SUPPORT}")
     if(${LIBOMP_OMPT_SUPPORT})
         say("LIBOMP: OMPT-blame           -- ${LIBOMP_OMPT_BLAME}")
         say("LIBOMP: OMPT-trace           -- ${LIBOMP_OMPT_TRACE}")
     endif()
     say("LIBOMP: Use build.pl rules   -- ${LIBOMP_USE_BUILDPL_RULES}")
     say("LIBOMP: Adaptive locks       -- ${LIBOMP_USE_ADAPTIVE_LOCKS}")
     say("LIBOMP: Use predefined linker flags      -- ${LIBOMP_USE_PREDEFINED_LINKER_FLAGS}")

runtime/cmake/Definitions.cmake

if(${DEBUG_BUILD})
         append_definitions("-D _DEBUG")
         append_definitions("-D BUILD_DEBUG")
     endif()
     if(${LIBOMP_STATS})
         append_definitions("-D KMP_STATS_ENABLED=1")
     else()
         append_definitions("-D KMP_STATS_ENABLED=0")
     endif()
+    if(${LIBOMP_USE_DEBUGGER})
+        append_definitions("-D USE_DEBUGGER=1")
+    else()
+        append_definitions("-D USE_DEBUGGER=0")
+    endif()
     if(${LIBOMP_OMPT_SUPPORT})
         append_definitions("-D OMPT_SUPPORT=1")
     else()
         append_definitions("-D OMPT_SUPPORT=0")
     endif()
     if(${LIBOMP_OMPT_BLAME})
         append_definitions("-D OMPT_BLAME=1")
     else()

runtime/cmake/SourceFiles.cmake

function(set_c_files input_c_source_files)
     endif()
     if(${STUBS_LIBRARY})
         append_c_source_file("kmp_stub.c")
     else()
         append_c_source_file("kmp_alloc.c")
         append_c_source_file("kmp_atomic.c")
         append_c_source_file("kmp_csupport.c")
         append_c_source_file("kmp_debug.c")
+        append_c_source_file("kmp_debugger.c")
         append_c_source_file("kmp_itt.c")
         append_c_source_file("kmp_environment.c")
         append_c_source_file("kmp_error.c")
         append_c_source_file("kmp_global.c")
         append_c_source_file("kmp_i18n.c")
         append_c_source_file("kmp_io.c")
         append_c_source_file("kmp_runtime.c")
         append_c_source_file("kmp_settings.c")

runtime/src/dllexports

# Regular entry points
         __kmp_acquire_nested_drdpa_lock
 
     %ifdef KMP_DEBUG
         # allows console output capability for applications those don't have it
         __kmp_printf
     %endif
 
 
+#if USE_DEBUGGER
+        __kmp_debugging                         DATA
+        __kmp_omp_debug_struct_info             DATA
+#endif /* USE_DEBUGGER */
 
         # Symbols for MS mutual detection:
         _You_must_link_with_exactly_one_OpenMP_library    DATA
         _You_must_link_with_Intel_OpenMP_library          DATA
         %ifdef msvc_compat
             _You_must_link_with_Microsoft_OpenMP_library  DATA
         %endif
 

runtime/src/exports_so.txt

global: # Exported symbols.
         kmp_*;     # Intel extensions.
         kmpc_*;    # Intel extensions.
         __kmpc_*;  # Functions called by compiler-generated code.
         GOMP_*;    # GNU C compatibility functions.
 
         _You_must_link_with_*;     # Mutual detection/MS compatibility symbols.
 
 
+        #
+        # Debugger support.
+        #
+#if USE_DEBUGGER
+        __kmp_debugging;
+        __kmp_omp_debug_struct_info;
+#endif /* USE_DEBUGGER */
 
         #
         # Internal functions exported for testing purposes.
         #
         __kmp_get_reduce_method;
         ___kmp_allocate;
         ___kmp_free;
         __kmp_thread_pool;

runtime/src/kmp.h

 
 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
 #include <xmmintrin.h>
 #endif
 
 #include "kmp_version.h"
 #include "kmp_debug.h"
 #include "kmp_lock.h"
+#if USE_DEBUGGER
+#include "kmp_debugger.h"
+#endif
 #include "kmp_i18n.h"
 
 #define KMP_HANDLE_SIGNALS (KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_WINDOWS || KMP_OS_DARWIN)
 
 #ifdef KMP_SETVERSION
 /*  from factory/Include, to get VERSION_STRING embedded for 'what'  */
 #include "kaiconfig.h"
 #include "eye.h"
typedef struct KMP_ALIGN_CACHE kmp_bstate {
     kmp_team_t *team;
     kmp_uint64 leaf_state;
     kmp_uint32 nproc;
     kmp_uint8 base_leaf_kids;
     kmp_uint8 leaf_kids;
     kmp_uint8 offset;
     kmp_uint8 wait_flag;
     kmp_uint8 use_oncore_barrier;
+#if USE_DEBUGGER
+    // The following field is intended for the debugger solely. Only the worker thread itself accesses this
+    // field: the worker increases it by 1 when it arrives to a barrier.
+    KMP_ALIGN_CACHE kmp_uint b_worker_arrived;
+#endif /* USE_DEBUGGER */
 } kmp_bstate_t;
 
 union KMP_ALIGN_CACHE kmp_barrier_union {
     double       b_align;        /* use worst case alignment */
     char         b_pad[ KMP_PAD(kmp_bstate_t, CACHE_LINE) ];
     kmp_bstate_t bb;
 };
 
 typedef union kmp_barrier_union kmp_balign_t;
 
 /* Team barrier needs only non-volatile arrived counter */
 union KMP_ALIGN_CACHE kmp_barrier_team_union {
     double       b_align;        /* use worst case alignment */
     char         b_pad[ CACHE_LINE ];
     struct {
         kmp_uint     b_arrived;       /* STATE => task reached synch point. */
+#if USE_DEBUGGER
+        // The following two fields are indended for the debugger solely. Only master of the team accesses
+        // these fields: the first one is increased by 1 when master arrives to a barrier, the
+        // second one is increased by one when all the threads arrived.
+        kmp_uint     b_master_arrived;
+        kmp_uint     b_team_arrived;
+#endif
     };
 };
 
 typedef union kmp_barrier_team_union kmp_balign_team_t;
 
 /*
  * Padding for Linux* OS pthreads condition variables and mutexes used to signal
  * threads when a condition changes.  This is to workaround an NPTL bug

 /* --------------------------------------------------------------------------- */
 
 extern kmp_global_t  __kmp_global;         /* global status */
 
 extern kmp_info_t __kmp_monitor;
 extern volatile kmp_uint32 __kmp_team_counter;      // Used by Debugging Support Library.
 extern volatile kmp_uint32 __kmp_task_counter;      // Used by Debugging Support Library.
 
+#if USE_DEBUGGER
+
 #define _KMP_GEN_ID( counter )                                         \
     (                                                                  \
+        __kmp_debugging                                                \
+        ?                                                              \
+        KMP_TEST_THEN_INC32( (volatile kmp_int32 *) & counter ) + 1    \
+        :                                                              \
         ~ 0                                                            \
     )
-
-
+#else
+#define _KMP_GEN_ID( counter )                                         \
+    (                                                                  \
+        ~ 0                                                            \
+    )
+#endif /* USE_DEBUGGER */
 
 #define KMP_GEN_TASK_ID()    _KMP_GEN_ID( __kmp_task_counter )
 #define KMP_GEN_TEAM_ID()    _KMP_GEN_ID( __kmp_team_counter )
 
 /* ------------------------------------------------------------------------ */
 /* ------------------------------------------------------------------------ */
 
 extern void __kmp_print_storage_map_gtid( int gtid, void *p1, void* p2, size_t size, char const *format, ... );

runtime/src/kmp_barrier.cpp

#endif /* USE_ITT_BUILD */
             this_thr->th.th_team_bt_intervals = team->t.t_implicit_task_taskdata[tid].td_icvs.bt_intervals;
             this_thr->th.th_team_bt_set = team->t.t_implicit_task_taskdata[tid].td_icvs.bt_set;
         }
 
 #if USE_ITT_BUILD
         if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
             __kmp_itt_barrier_starting(gtid, itt_sync_obj);
 #endif /* USE_ITT_BUILD */
-
+#if USE_DEBUGGER
+        // Let the debugger know: the thread arrived to the barrier and waiting.
+        if (KMP_MASTER_TID(tid)) { // Master counter is stored in team structure.
+            team->t.t_bar[bt].b_master_arrived += 1;
+        } else {
+            this_thr->th.th_bar[bt].bb.b_worker_arrived += 1;
+        } // if
+#endif /* USE_DEBUGGER */
         if (reduce != NULL) {
             //KMP_DEBUG_ASSERT( is_split == TRUE );  // #C69956
             this_thr->th.th_local.reduce_data = reduce_data;
         }
         switch (__kmp_barrier_gather_pattern[bt]) {
         case bp_hyper_bar: {
             KMP_ASSERT(__kmp_barrier_gather_branch_bits[bt]); // don't set branch bits to 0; use linear
             __kmp_hyper_barrier_gather(bt, this_thr, gtid, tid, reduce
#endif /* USE_DEBUGGER */
 
         if (KMP_MASTER_TID(tid)) {
             status = 0;
             if (__kmp_tasking_mode != tskm_immediate_exec) {
                 __kmp_task_team_wait(this_thr, team
                                      USE_ITT_BUILD_ARG(itt_sync_obj) );
                 __kmp_task_team_setup(this_thr, team, 0, 0); // use 0,0 to only setup the current team if nthreads > 1
             }
-
+#if USE_DEBUGGER
+            // Let the debugger know: All threads are arrived and starting leaving the barrier.
+            team->t.t_bar[bt].b_team_arrived += 1;
+#endif
 
 #if USE_ITT_BUILD
             /* TODO: In case of split reduction barrier, master thread may send acquired event early,
                before the final summation into the shared variable is done (final summation can be a
                long operation for array reductions).  */
             if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
                 __kmp_itt_barrier_middle(gtid, itt_sync_obj);
 #endif /* USE_ITT_BUILD */

runtime/src/kmp_debugger.h

+#if USE_DEBUGGER
+/*
+ * kmp_debugger.h -- debugger support.
+ */
+
+
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.txt for details.
+//
+//===----------------------------------------------------------------------===//
+
+
+#ifndef KMP_DEBUGGER_H
+#define KMP_DEBUGGER_H
+
+#ifdef __cplusplus
+    extern "C" {
+#endif // __cplusplus
+
+extern int __kmp_debugging;             /* Boolean whether currently debugging OpenMP RTL */
+// Return number of threads specified by the debugger for given parallel region.
+int __kmp_omp_num_threads( ident_t const * ident );
+
+#ifdef __cplusplus
+    } // extern "C"
+#endif // __cplusplus
+
+
+#endif // KMP_DEBUGGER_H
+
+#endif // USE_DEBUGGER

runtime/src/kmp_debugger.c

+#if USE_DEBUGGER
+/*
+ * kmp_debugger.c -- debugger support.
+ */
+
+
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.txt for details.
+//
+//===----------------------------------------------------------------------===//
+
+
+#include "kmp.h"
+#include "kmp_lock.h"
+#include "kmp_omp.h"
+#include "kmp_str.h"
+
+/*
+    NOTE: All variable names are known to the debugger, do not change!
+*/
+
+#ifdef __cplusplus
+    extern "C" {
+        extern kmp_omp_struct_info_t __kmp_omp_debug_struct_info;
+    } // extern "C"
+#endif // __cplusplus
+
+int __kmp_debugging          = FALSE;    // Boolean whether currently debugging OpenMP RTL.
+
+#define offset_and_size_of( structure, field )     \
+    {                                              \
+        offsetof( structure,           field ),    \
+        sizeof( ( (structure *) NULL)->field )     \
+    }
+
+#define offset_and_size_not_available \
+    { -1, -1 }
+
+#define addr_and_size_of( var )                    \
+    {                                              \
+        (kmp_uint64)( & var ),                     \
+        sizeof( var )                              \
+    }
+
+#define nthr_buffer_size 1024
+static kmp_int32
+kmp_omp_nthr_info_buffer[ nthr_buffer_size ] =
+    { nthr_buffer_size * sizeof( kmp_int32 ) };
+
+/* TODO: Check punctuation for various platforms here */
+static char func_microtask[]    = "__kmp_invoke_microtask";
+static char func_fork[]         = "__kmpc_fork_call";
+static char func_fork_teams[]   = "__kmpc_fork_teams";
+
+
+// Various info about runtime structures: addresses, field offsets, sizes, etc.
+kmp_omp_struct_info_t
+__kmp_omp_debug_struct_info = {
+
+    /* Change this only if you make a fundamental data structure change here */
+    KMP_OMP_VERSION,
+
+    /* sanity check.  Only should be checked if versions are identical
+     * This is also used for backward compatibility to get the runtime
+     * structure size if it the runtime is older than the interface */
+    sizeof( kmp_omp_struct_info_t ),
+
+    /* OpenMP RTL version info. */
+    addr_and_size_of( __kmp_version_major ),
+    addr_and_size_of( __kmp_version_minor ),
+    addr_and_size_of( __kmp_version_build ),
+    addr_and_size_of( __kmp_openmp_version ),
+    { (kmp_uint64)( __kmp_copyright ) + KMP_VERSION_MAGIC_LEN, 0 },        // Skip magic prefix.
+
+    /* Various globals. */
+    addr_and_size_of( __kmp_threads ),
+    addr_and_size_of( __kmp_root ),
+    addr_and_size_of( __kmp_threads_capacity ),
+    addr_and_size_of( __kmp_monitor ),
+#if ! KMP_USE_DYNAMIC_LOCK
+    addr_and_size_of( __kmp_user_lock_table ),
+#endif
+    addr_and_size_of( func_microtask ),
+    addr_and_size_of( func_fork ),
+    addr_and_size_of( func_fork_teams ),
+    addr_and_size_of( __kmp_team_counter ),
+    addr_and_size_of( __kmp_task_counter ),
+    addr_and_size_of( kmp_omp_nthr_info_buffer ),
+    sizeof( void * ),
+    OMP_LOCK_T_SIZE < sizeof(void *),
+    bs_last_barrier,
+    TASK_DEQUE_SIZE,
+
+    // thread structure information
+    sizeof( kmp_base_info_t ),
+    offset_and_size_of( kmp_base_info_t, th_info ),
+    offset_and_size_of( kmp_base_info_t, th_team ),
+    offset_and_size_of( kmp_base_info_t, th_root ),
+    offset_and_size_of( kmp_base_info_t, th_serial_team ),
+    offset_and_size_of( kmp_base_info_t, th_ident ),
+    offset_and_size_of( kmp_base_info_t, th_spin_here    ),
+    offset_and_size_of( kmp_base_info_t, th_next_waiting ),
+    offset_and_size_of( kmp_base_info_t, th_task_team    ),
+    offset_and_size_of( kmp_base_info_t, th_current_task ),
+    offset_and_size_of( kmp_base_info_t, th_task_state   ),
+    offset_and_size_of( kmp_base_info_t,   th_bar ),
+    offset_and_size_of( kmp_bstate_t,      b_worker_arrived ),
+
+    // teams information
+    offset_and_size_of( kmp_base_info_t, th_teams_microtask),
+    offset_and_size_of( kmp_base_info_t, th_teams_level),
+    offset_and_size_of( kmp_teams_size_t, nteams ),
+    offset_and_size_of( kmp_teams_size_t, nth ),
+
+    // kmp_desc structure (for info field above)
+    sizeof( kmp_desc_base_t ),
+    offset_and_size_of( kmp_desc_base_t, ds_tid    ),
+    offset_and_size_of( kmp_desc_base_t, ds_gtid   ),
+    offset_and_size_of(
+        kmp_desc_base_t,
+        // On Windows* OS, ds_thread contains a thread /handle/, which is not usable, while thread /id/
+        // is in ds_thread_id.
+        #if KMP_OS_WINDOWS
+            ds_thread_id
+        #else
+            ds_thread
+        #endif
+    ),
+
+    // team structure information
+    sizeof( kmp_base_team_t ),
+    offset_and_size_of( kmp_base_team_t,   t_master_tid ),
+    offset_and_size_of( kmp_base_team_t,   t_ident      ),
+    offset_and_size_of( kmp_base_team_t,   t_parent     ),
+    offset_and_size_of( kmp_base_team_t,   t_nproc      ),
+    offset_and_size_of( kmp_base_team_t,   t_threads    ),
+    offset_and_size_of( kmp_base_team_t,   t_serialized ),
+    offset_and_size_of( kmp_base_team_t,   t_id         ),
+    offset_and_size_of( kmp_base_team_t,   t_pkfn       ),
+    offset_and_size_of( kmp_base_team_t,   t_task_team ),
+    offset_and_size_of( kmp_base_team_t,   t_implicit_task_taskdata ),
+    offset_and_size_of( kmp_base_team_t,   t_cancel_request ),
+    offset_and_size_of( kmp_base_team_t,   t_bar ),
+    offset_and_size_of( kmp_balign_team_t, b_master_arrived ),
+    offset_and_size_of( kmp_balign_team_t, b_team_arrived ),
+
+    // root structure information
+    sizeof( kmp_base_root_t ),
+    offset_and_size_of( kmp_base_root_t, r_root_team   ),
+    offset_and_size_of( kmp_base_root_t, r_hot_team    ),
+    offset_and_size_of( kmp_base_root_t, r_uber_thread ),
+    offset_and_size_not_available,
+
+    // ident structure information
+    sizeof( ident_t ),
+    offset_and_size_of( ident_t, psource ),
+    offset_and_size_of( ident_t, flags   ),
+
+    // lock structure information
+    sizeof( kmp_base_queuing_lock_t ),
+    offset_and_size_of( kmp_base_queuing_lock_t, initialized  ),
+    offset_and_size_of( kmp_base_queuing_lock_t, location ),
+    offset_and_size_of( kmp_base_queuing_lock_t, tail_id  ),
+    offset_and_size_of( kmp_base_queuing_lock_t, head_id  ),
+    offset_and_size_of( kmp_base_queuing_lock_t, next_ticket  ),
+    offset_and_size_of( kmp_base_queuing_lock_t, now_serving  ),
+    offset_and_size_of( kmp_base_queuing_lock_t, owner_id     ),
+    offset_and_size_of( kmp_base_queuing_lock_t, depth_locked ),
+    offset_and_size_of( kmp_base_queuing_lock_t, flags ),
+
+#if ! KMP_USE_DYNAMIC_LOCK
+    /* Lock table. */
+    sizeof( kmp_lock_table_t ),
+    offset_and_size_of( kmp_lock_table_t, used       ),
+    offset_and_size_of( kmp_lock_table_t, allocated  ),
+    offset_and_size_of( kmp_lock_table_t, table      ),
+#endif
+
+    // Task team structure information.
+    sizeof( kmp_base_task_team_t ),
+    offset_and_size_of( kmp_base_task_team_t, tt_threads_data       ),
+    offset_and_size_of( kmp_base_task_team_t, tt_found_tasks        ),
+    offset_and_size_of( kmp_base_task_team_t, tt_nproc              ),
+    offset_and_size_of( kmp_base_task_team_t, tt_unfinished_threads ),
+    offset_and_size_of( kmp_base_task_team_t, tt_active             ),
+
+    // task_data_t.
+    sizeof( kmp_taskdata_t ),
+    offset_and_size_of( kmp_taskdata_t, td_task_id                ),
+    offset_and_size_of( kmp_taskdata_t, td_flags                  ),
+    offset_and_size_of( kmp_taskdata_t, td_team                   ),
+    offset_and_size_of( kmp_taskdata_t, td_parent                 ),
+    offset_and_size_of( kmp_taskdata_t, td_level                  ),
+    offset_and_size_of( kmp_taskdata_t, td_ident                  ),
+    offset_and_size_of( kmp_taskdata_t, td_allocated_child_tasks  ),
+    offset_and_size_of( kmp_taskdata_t, td_incomplete_child_tasks ),
+
+    offset_and_size_of( kmp_taskdata_t, td_taskwait_ident   ),
+    offset_and_size_of( kmp_taskdata_t, td_taskwait_counter ),
+    offset_and_size_of( kmp_taskdata_t, td_taskwait_thread  ),
+
+    offset_and_size_of( kmp_taskdata_t, td_taskgroup        ),
+    offset_and_size_of( kmp_taskgroup_t, count              ),
+    offset_and_size_of( kmp_taskgroup_t, cancel_request     ),
+
+    offset_and_size_of( kmp_taskdata_t, td_depnode          ),
+    offset_and_size_of( kmp_depnode_list_t, node            ),
+    offset_and_size_of( kmp_depnode_list_t, next            ),
+    offset_and_size_of( kmp_base_depnode_t, successors      ),
+    offset_and_size_of( kmp_base_depnode_t, task            ),
+    offset_and_size_of( kmp_base_depnode_t, npredecessors   ),
+    offset_and_size_of( kmp_base_depnode_t, nrefs           ),
+    offset_and_size_of( kmp_task_t, routine                 ),
+
+    // thread_data_t.
+    sizeof( kmp_thread_data_t ),
+    offset_and_size_of( kmp_base_thread_data_t, td_deque             ),
+    offset_and_size_of( kmp_base_thread_data_t, td_deque_head        ),
+    offset_and_size_of( kmp_base_thread_data_t, td_deque_tail        ),
+    offset_and_size_of( kmp_base_thread_data_t, td_deque_ntasks      ),
+    offset_and_size_of( kmp_base_thread_data_t, td_deque_last_stolen ),
+
+    // The last field.
+    KMP_OMP_VERSION,
+
+}; // __kmp_omp_debug_struct_info
+
+#undef offset_and_size_of
+#undef addr_and_size_of
+
+/*
+  Intel compiler on IA-32 architecture issues a warning "conversion
+  from "unsigned long long" to "char *" may lose significant bits"
+  when 64-bit value is assigned to 32-bit pointer. Use this function
+  to suppress the warning.
+*/
+static inline
+void *
+__kmp_convert_to_ptr(
+    kmp_uint64    addr
+) {
+    #if KMP_COMPILER_ICC
+        #pragma warning( push )
+        #pragma warning( disable:  810 ) // conversion from "unsigned long long" to "char *" may lose significant bits
+        #pragma warning( disable: 1195 ) // conversion from integer to smaller pointer
+    #endif // KMP_COMPILER_ICC
+    return (void *) addr;
+    #if KMP_COMPILER_ICC
+        #pragma warning( pop )
+    #endif // KMP_COMPILER_ICC
+} // __kmp_convert_to_ptr
+
+
+static int
+kmp_location_match(
+    kmp_str_loc_t *        loc,
+    kmp_omp_nthr_item_t *  item
+) {
+
+    int file_match = 0;
+    int func_match = 0;
+    int line_match = 0;
+
+    char * file = (char *) __kmp_convert_to_ptr( item->file );
+    char * func = (char *) __kmp_convert_to_ptr( item->func );
+    file_match = __kmp_str_fname_match( & loc->fname, file );
+    func_match =
+        item->func == 0  // If item->func is NULL, it allows any func name.
+        ||
+        strcmp( func, "*" ) == 0
+        ||
+        ( loc->func != NULL && strcmp( loc->func, func ) == 0 );
+    line_match =
+        item->begin <= loc->line
+        &&
+        ( item->end <= 0 || loc->line <= item->end ); // if item->end <= 0, it means "end of file".
+
+    return ( file_match && func_match && line_match );
+
+} // kmp_location_match
+
+
+int
+__kmp_omp_num_threads(
+    ident_t const * ident
+) {
+
+    int num_threads = 0;
+
+    kmp_omp_nthr_info_t * info =
+        (kmp_omp_nthr_info_t *) __kmp_convert_to_ptr(  __kmp_omp_debug_struct_info.nthr_info.addr );
+    if ( info->num > 0 && info->array != 0 ) {
+        kmp_omp_nthr_item_t * items = (kmp_omp_nthr_item_t *) __kmp_convert_to_ptr( info->array );
+        kmp_str_loc_t         loc   = __kmp_str_loc_init( ident->psource, 1 );
+        int i;
+        for ( i = 0; i < info->num; ++ i ) {
+            if ( kmp_location_match( & loc, & items[ i ] ) ) {
+                num_threads = items[ i ].num_threads;
+            }; // if
+        }; // for
+        __kmp_str_loc_free( & loc );
+    }; // if
+
+    return num_threads;;
+
+} // __kmp_omp_num_threads
+#endif /* USE_DEBUGGER */

runtime/src/kmp_omp.h

+#if USE_DEBUGGER
 /*
  * kmp_omp.h -- OpenMP definition for kmp_omp_struct_info_t.
  *              This is for information about runtime library structures.
  */
 
 
 //===----------------------------------------------------------------------===//
 //
#endif
 
     /* Taskgroup */
     offset_and_size_t  td_taskgroup;                // pointer to the current taskgroup
     offset_and_size_t  td_task_count;               // number of allocated and not yet complete tasks
     offset_and_size_t  td_cancel;                   // request for cancellation of this taskgroup
 
     /* Task dependency */
     offset_and_size_t  td_depnode;                  // pointer to graph node if the task has dependencies
-    offset_and_size_t  dn_successors;
     offset_and_size_t  dn_node;
     offset_and_size_t  dn_next;
+    offset_and_size_t  dn_successors;
     offset_and_size_t  dn_task;
     offset_and_size_t  dn_npredecessors;
     offset_and_size_t  dn_nrefs;
+    offset_and_size_t  dn_routine;
 
     /* kmp_thread_data_t */
     kmp_int32          hd_sizeof_struct;
     offset_and_size_t  hd_deque;
     offset_and_size_t  hd_deque_head;
     offset_and_size_t  hd_deque_tail;
     offset_and_size_t  hd_deque_ntasks;
     offset_and_size_t  hd_deque_last_stolen;
 
     // The last field of stable version.
     kmp_uint64         last_field;
 
 } kmp_omp_struct_info_t;
 
+#endif /* USE_DEBUGGER */
 
 /* end of file */

runtime/src/kmp_runtime.c

#if OMP_40_ENABLED
             thr->th.th_teams_size      = master_th->th.th_teams_size;
 #endif
             { // Initialize threads' barrier data.
                 int b;
                 kmp_balign_t * balign = team->t.t_threads[ i ]->th.th_bar;
                 for ( b = 0; b < bs_last_barrier; ++ b ) {
                     balign[ b ].bb.b_arrived        = team->t.t_bar[ b ].b_arrived;
                     KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
+#if USE_DEBUGGER
+                    balign[ b ].bb.b_worker_arrived = team->t.t_bar[ b ].b_team_arrived;
+#endif
                 }; // for b
             }
         }
 
 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED
         __kmp_partition_places( team );
 #endif
 

 
 #if OMP_40_ENABLED
         if ( __kmp_nested_proc_bind.used && ( level + 1 < __kmp_nested_proc_bind.used ) ) {
             this_thr->th.th_current_task->td_icvs.proc_bind
                 = __kmp_nested_proc_bind.bind_types[ level + 1 ];
         }
 #endif /* OMP_40_ENABLED */
 
+#if USE_DEBUGGER
+        serial_team->t.t_pkfn = (microtask_t)( ~0 ); // For the debugger.
+#endif
         this_thr->th.th_info.ds.ds_tid = 0;
 
         /* set thread cache values */
         this_thr->th.th_team_nproc     = 1;
         this_thr->th.th_team_master    = this_thr;
         this_thr->th.th_team_serialized = 1;
 
         serial_team->t.t_level        = serial_team->t.t_parent->t.t_level + 1;
#endif
                 for ( i = 0; i < master_set_numthreads; ++i ) {
                     other_threads[i]->th.th_team_nproc = master_set_numthreads;
                 }
                 // Keep extra threads hot in the team for possible next parallels
             }
             master_th->th.th_set_nproc = 0;
         }
 
+#if USE_DEBUGGER
+    if ( __kmp_debugging ) {    // Let debugger override number of threads.
+        int nth = __kmp_omp_num_threads( loc );
+        if ( nth > 0 ) {        // 0 means debugger does not want to change number of threads.
+            master_set_numthreads = nth;
+        }; // if
+    }; // if
+#endif
 
         KF_TRACE( 10, ( "__kmp_fork_call: before internal fork: root=%p, team=%p, master_th=%p, gtid=%d\n", root, parent_team, master_th, gtid ) );
         __kmp_internal_fork( loc, gtid, parent_team );
         KF_TRACE( 10, ( "__kmp_fork_call: after internal fork: root=%p, team=%p, master_th=%p, gtid=%d\n", root, parent_team, master_th, gtid ) );
 
         /* Invoke microtask for MASTER thread */
         KA_TRACE( 20, ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n",
                     gtid, parent_team->t.t_id, parent_team->t.t_pkfn ) );
if ( master_th->th.th_teams_microtask &&
             // Adjust states of non-used threads of the team
             for ( i = old_num; i < new_num; ++i ) {
                 // Re-initialize thread's barrier data.
                 int b;
                 kmp_balign_t * balign = other_threads[i]->th.th_bar;
                 for ( b = 0; b < bs_last_barrier; ++ b ) {
                     balign[ b ].bb.b_arrived        = team->t.t_bar[ b ].b_arrived;
                     KMP_DEBUG_ASSERT(balign[ b ].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
+#if USE_DEBUGGER
+                    balign[ b ].bb.b_worker_arrived = team->t.t_bar[ b ].b_team_arrived;
+#endif
                 }
                 if ( __kmp_tasking_mode != tskm_immediate_exec ) {
                     // Synchronize thread's task state
                     other_threads[i]->th.th_task_state = master_th->th.th_task_state;
                 }
             }
         }
 

 #endif
 #if OMP_40_ENABLED
             __kmp_nested_proc_bind.bind_types[0],
 #endif
             &r_icvs,
             0                                                          // argc
             USE_NESTED_HOT_ARG(NULL)                                   // master thread is unknown
         );
+#if USE_DEBUGGER
+    // Non-NULL value should be assigned to make the debugger display the root team.
+    TCW_SYNC_PTR(root_team->t.t_pkfn, (microtask_t)( ~ 0 ));
+#endif
 
     KF_TRACE( 10, ( "__kmp_initialize_root: after root_team = %p\n", root_team ) );
 
     root->r.r_root_team = root_team;
     root_team->t.t_control_stack_top = NULL;
 
     /* initialize root team */
     root_team->t.t_threads[0] = NULL;
#endif
     KA_TRACE( 20, ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, plain=%u\n",
                     gtid, __kmp_gtid_from_tid( 0, root->r.r_hot_team ),
                     root->r.r_hot_team->t.t_id, 0, KMP_INIT_BARRIER_STATE,
                     KMP_INIT_BARRIER_STATE ) );
     { // Initialize barrier data.
         int b;
         for ( b = 0; b < bs_last_barrier; ++ b ) {
             root_thread->th.th_bar[ b ].bb.b_arrived        = KMP_INIT_BARRIER_STATE;
+#if USE_DEBUGGER
+            root_thread->th.th_bar[ b ].bb.b_worker_arrived = 0;
+#endif
         }; // for
     }
     KMP_DEBUG_ASSERT( root->r.r_hot_team->t.t_bar[ bs_forkjoin_barrier ].b_arrived == KMP_INIT_BARRIER_STATE );
 
 
 #if KMP_AFFINITY_SUPPORTED
     if ( TCR_4(__kmp_init_middle) ) {
         __kmp_affinity_set_init_mask( gtid, TRUE );
#if KMP_NESTED_HOT_TEAMS
             for ( f = team->t.t_nproc; f < avail_threads; ++f ) {
                 // Adjust barrier data of reserved threads (if any) of the team
                 // Other data will be set in __kmp_initialize_info() below.
                 int b;
                 kmp_balign_t * balign = other_threads[f]->th.th_bar;
                 for ( b = 0; b < bs_last_barrier; ++ b ) {
                     balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
                     KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
+#if USE_DEBUGGER
+                    balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
+#endif
                 }
             }
             if( hot_teams[level].hot_team_nth >= new_nproc ) {
                 // we have all needed threads in reserve, no need to allocate any
                 // this only possible in mode 1, cannot have reserved threads in mode 0
                 KMP_DEBUG_ASSERT(__kmp_hot_teams_mode == 1);
                 team->t.t_nproc = new_nproc;                     // just get reserved threads involved
             } else {
#endif
                                 team->t.t_bar[bs_plain_barrier].b_arrived ) );
 
                 { // Initialize barrier data for new threads.
                     int b;
                     kmp_balign_t * balign = new_worker->th.th_bar;
                     for( b = 0; b < bs_last_barrier; ++ b ) {
                         balign[ b ].bb.b_arrived        = team->t.t_bar[ b ].b_arrived;
                         KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
+#if USE_DEBUGGER
+                        balign[ b ].bb.b_worker_arrived = team->t.t_bar[ b ].b_team_arrived;
+#endif
                     }
                 }
             }
 
 #if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED
             if ( KMP_AFFINITY_CAPABLE() ) {
                 /* Restore initial master thread's affinity mask */
                 __kmp_set_system_affinity( old_mask, TRUE );
#if KMP_NESTED_HOT_TEAMS
             for( f = 1; f < new_nproc; ++f ) {
                 kmp_info_t *thr = team->t.t_threads[f];
                 thr->th.th_task_state = 0;
                 int b;
                 kmp_balign_t * balign = thr->th.th_bar;
                 for( b = 0; b < bs_last_barrier; ++ b ) {
                     balign[ b ].bb.b_arrived        = team->t.t_bar[ b ].b_arrived;
                     KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
+#if USE_DEBUGGER
+                    balign[ b ].bb.b_worker_arrived = team->t.t_bar[ b ].b_team_arrived;
+#endif
                 }
             }
         }
 #endif // KMP_NESTED_HOT_TEAMS
 
         /* reallocate space for arguments if necessary */
         __kmp_alloc_argv_entries( argc, team, TRUE );
         team->t.t_argc     = argc;
for( team = (kmp_team_t*) __kmp_team_pool ; (team) ; )
             team->t.t_argc     = argc;
 
             KA_TRACE( 20, ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n",
                             team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE ));
             { // Initialize barrier data.
                 int b;
                 for ( b = 0; b < bs_last_barrier; ++ b) {
                     team->t.t_bar[ b ].b_arrived        = KMP_INIT_BARRIER_STATE;
+#if USE_DEBUGGER
+                    team->t.t_bar[ b ].b_master_arrived = 0;
+                    team->t.t_bar[ b ].b_team_arrived   = 0;
+#endif
                 }
             }
 
 #if OMP_40_ENABLED
             team->t.t_proc_bind = new_proc_bind;
 #endif
 
             KA_TRACE( 20, ("__kmp_allocate_team: using team from pool %d.\n", team->t.t_id ));
#endif
     team->t.t_argc        = argc;
 
     KA_TRACE( 20, ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n",
                     team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE ));
     { // Initialize barrier data.
         int b;
         for ( b = 0; b < bs_last_barrier; ++ b ) {
             team->t.t_bar[ b ].b_arrived        = KMP_INIT_BARRIER_STATE;
+#if USE_DEBUGGER
+            team->t.t_bar[ b ].b_master_arrived = 0;
+            team->t.t_bar[ b ].b_team_arrived   = 0;
+#endif
         }
     }
 
 #if OMP_40_ENABLED
     team->t.t_proc_bind = new_proc_bind;
 #endif
 
 #if OMPT_SUPPORT

runtime/src/makefile.mk

 ifeq "$(LIB_TYPE)" "stub"
     lib_c_items += kmp_stub
 else # norm or prof
     lib_c_items +=                   \
         kmp_alloc                    \
         kmp_atomic                   \
         kmp_csupport                 \
         kmp_debug                    \
-  kmp_itt                      \
+        kmp_debugger                 \
+        kmp_itt                      \
         $(empty)
     ifeq "$(USE_ITT_NOTIFY)" "1"
         lib_c_items +=  ittnotify_static
     endif
 
 
     lib_cpp_items +=                 \
         kmp_environment              \

runtime/tools/src/common-tools.mk

 # For non-x86 architecture
 ifeq "$(filter 32 32e 64 mic,$(arch))" ""
     cpp-flags += $(shell pkg-config --cflags libffi)
 endif
 # Add all VPATH directories to path for searching include files.
 cpp-flags += $(foreach i,$(VPATH),-I $(i))
 
 
+cpp-flags += -D USE_DEBUGGER=1
+
 # Shouldn't this be being set from the command line somehow?
 cpp-flags += -D USE_ITT_BUILD
 
 ifeq "$(OPTIMIZATION)" "on"
     cpp-flags += -D NDEBUG
 else
     cpp-flags += -D _DEBUG -D BUILD_DEBUG
     ifeq "$(os)" "win"