Hinted lock (OpenMP 4.5 feature) Updates/Fixes

[jlpeyton]

Summary

There are going to be two more patches which bring this feature up to date and in line with OpenMP 4.5.

1. Renamed jump tables for the lock functions (and some clean up).
2. Renamed some macros to be in KMP_ namespace.
3. Return type of unset functions changed from void to int.
4. Enabled use of _xebgin() et al. intrinsics for accessing TSX instructions.

[jlpeyton]

Adding Hansang as reviewer.

[hbae]

LGTM.

openmp/trunk/runtime/src/kmp_csupport.c

if (tag == locktag_tas && !__kmp_env_consistency_check) {
         KMP_ACQUIRE_TAS_LOCK(user_lock, gtid);
     } else
 # elif KMP_USE_FAST_FUTEX
     if (tag == locktag_futex && !__kmp_env_consistency_check) {
         KMP_ACQUIRE_FUTEX_LOCK(user_lock, gtid);
     } else
 # endif
     {
-        __kmp_direct_set_ops[tag]((kmp_dyna_lock_t *)user_lock, gtid);
+        __kmp_direct_set[tag]((kmp_dyna_lock_t *)user_lock, gtid);
     }
 # if USE_ITT_BUILD
     __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
 # endif
 
 #else // KMP_USE_DYNAMIC_LOCK
 
     kmp_user_lock_p lck;
if (tag == locktag_tas && !__kmp_env_consistency_check) {
         KMP_RELEASE_TAS_LOCK(user_lock, gtid);
     } else
 # elif KMP_USE_FAST_FUTEX
     if (tag == locktag_futex && !__kmp_env_consistency_check) {
         KMP_RELEASE_FUTEX_LOCK(user_lock, gtid);
     } else
 # endif
     {
-        __kmp_direct_unset_ops[tag]((kmp_dyna_lock_t *)user_lock, gtid);
+        __kmp_direct_unset[tag]((kmp_dyna_lock_t *)user_lock, gtid);
     }
 
 #else // KMP_USE_DYNAMIC_LOCK
 
     kmp_user_lock_p lck;
 
     /* Can't use serial interval since not block structured */
     /* release the lock */
if (tag == locktag_tas && !__kmp_env_consistency_check) {
         KMP_TEST_TAS_LOCK(user_lock, gtid, rc);
     } else
 # elif KMP_USE_FAST_FUTEX
     if (tag == locktag_futex && !__kmp_env_consistency_check) {
         KMP_TEST_FUTEX_LOCK(user_lock, gtid, rc);
     } else
 # endif
     {
-        rc = __kmp_direct_test_ops[tag]((kmp_dyna_lock_t *)user_lock, gtid);
+        rc = __kmp_direct_test[tag]((kmp_dyna_lock_t *)user_lock, gtid);
     }
     if (rc) {
 # if USE_ITT_BUILD
         __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
 # endif
         return FTN_TRUE;
     } else {
 # if USE_ITT_BUILD

openmp/trunk/runtime/src/kmp_lock.h

 #define KMP_USE_FAST_FUTEX     0 && KMP_HAS_FUTEX
 #define KMP_USE_FAST_TAS       1 && KMP_HAS_FUTEX
 
 // List of lock definitions; all nested locks are indirect locks.
 // hle lock is xchg lock prefixed with XACQUIRE/XRELEASE.
 // All nested locks are indirect lock types.
 #if KMP_HAS_FUTEX
 # if KMP_HAS_HLE
-#  define FOREACH_D_LOCK(m, a) m(tas, a) m(futex, a) m(hle, a)
-#  define KMP_LAST_D_LOCK_SEQ lockseq_hle
+#  define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(futex, a) m(hle, a)
+#  define KMP_LAST_D_LOCK lockseq_hle
 # else
-#  define FOREACH_D_LOCK(m, a) m(tas, a) m(futex, a)
-#  define KMP_LAST_D_LOCK_SEQ lockseq_futex
+#  define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(futex, a)
+#  define KMP_LAST_D_LOCK lockseq_futex
 # endif // KMP_HAS_HLE
 # if KMP_USE_ADAPTIVE_LOCKS
-#  define FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(adaptive, a) m(drdpa, a)   \
+#  define KMP_FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(adaptive, a) m(drdpa, a)   \
                                m(nested_tas, a) m(nested_futex, a) m(nested_ticket, a) \
                                m(nested_queuing, a) m(nested_drdpa, a)
 # else
-#  define FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a)                m(drdpa, a)   \
+#  define KMP_FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a)                m(drdpa, a)   \
                                m(nested_tas, a) m(nested_futex, a) m(nested_ticket, a) \
                                m(nested_queuing, a) m(nested_drdpa, a)
 # endif // KMP_USE_ADAPTIVE_LOCKS
 #else
 # if KMP_HAS_HLE
-#  define FOREACH_D_LOCK(m, a) m(tas, a)             m(hle, a)
-#  define KMP_LAST_D_LOCK_SEQ lockseq_hle
+#  define KMP_FOREACH_D_LOCK(m, a) m(tas, a)             m(hle, a)
+#  define KMP_LAST_D_LOCK lockseq_hle
 # else
-#  define FOREACH_D_LOCK(m, a) m(tas, a)
-#  define KMP_LAST_D_LOCK_SEQ lockseq_tas
+#  define KMP_FOREACH_D_LOCK(m, a) m(tas, a)
+#  define KMP_LAST_D_LOCK lockseq_tas
 # endif // KMP_HAS_HLE
 # if KMP_USE_ADAPTIVE_LOCKS
-#  define FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(adaptive, a) m(drdpa, a)   \
+#  define KMP_FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(adaptive, a) m(drdpa, a)   \
                                m(nested_tas, a)                    m(nested_ticket, a) \
                                m(nested_queuing, a) m(nested_drdpa, a)
 # else
-#  define FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a)                m(drdpa, a)   \
+#  define KMP_FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a)                m(drdpa, a)   \
                                m(nested_tas, a)                    m(nested_ticket, a) \
                                m(nested_queuing, a) m(nested_drdpa, a)
 # endif // KMP_USE_ADAPTIVE_LOCKS
 #endif // KMP_HAS_FUTEX
 
 // Information used in dynamic dispatch
-#define KMP_LOCK_VALUE_SHIFT 8
-#define KMP_LOCK_TYPE_MASK   ((1<<KMP_LOCK_VALUE_SHIFT)-1)
-#define KMP_NUM_D_LOCKS      KMP_LAST_D_LOCK_SEQ
-#define KMP_NUM_I_LOCKS      (locktag_nested_drdpa+1)
+#define KMP_LOCK_SHIFT   8 // number of low bits to be used as tag for direct locks
+#define KMP_FIRST_D_LOCK lockseq_tas
+#define KMP_FIRST_I_LOCK lockseq_ticket
+#define KMP_LAST_I_LOCK  lockseq_nested_drdpa
+#define KMP_NUM_I_LOCKS  (locktag_nested_drdpa+1) // number of indirect lock types
 
 // Base type for dynamic locks.
 typedef kmp_uint32 kmp_dyna_lock_t;
 
 // Lock sequence that enumerates all lock kinds.
 // Always make this enumeration consistent with kmp_lockseq_t in the include directory.
 typedef enum {
     lockseq_indirect = 0,
 #define expand_seq(l,a) lockseq_##l,
-    FOREACH_D_LOCK(expand_seq, 0)
-    FOREACH_I_LOCK(expand_seq, 0)
+    KMP_FOREACH_D_LOCK(expand_seq, 0)
+    KMP_FOREACH_I_LOCK(expand_seq, 0)
 #undef expand_seq
 } kmp_dyna_lockseq_t;
 
 // Enumerates indirect lock tags.
 typedef enum {
 #define expand_tag(l,a) locktag_##l,
-    FOREACH_I_LOCK(expand_tag, 0)
+    KMP_FOREACH_I_LOCK(expand_tag, 0)
 #undef expand_tag
 } kmp_indirect_locktag_t;
 
 // Utility macros that extract information from lock sequences.
-#define KMP_IS_D_LOCK(seq) (seq >= lockseq_tas && seq <= KMP_LAST_D_LOCK_SEQ)
-#define KMP_IS_I_LOCK(seq) (seq >= lockseq_ticket && seq <= lockseq_nested_drdpa)
-#define KMP_GET_I_TAG(seq) (kmp_indirect_locktag_t)(seq - lockseq_ticket)
-#define KMP_GET_D_TAG(seq) (seq<<1 | 1)
+#define KMP_IS_D_LOCK(seq) ((seq) >= KMP_FIRST_D_LOCK && (seq) <= KMP_LAST_D_LOCK)
+#define KMP_IS_I_LOCK(seq) ((seq) >= KMP_FIRST_I_LOCK && (seq) <= KMP_LAST_I_LOCK)
+#define KMP_GET_I_TAG(seq) (kmp_indirect_locktag_t)((seq) - KMP_FIRST_I_LOCK)
+#define KMP_GET_D_TAG(seq) ((seq)<<1 | 1)
 
 // Enumerates direct lock tags starting from indirect tag.
 typedef enum {
 #define expand_tag(l,a) locktag_##l = KMP_GET_D_TAG(lockseq_##l),
-    FOREACH_D_LOCK(expand_tag, 0)
+    KMP_FOREACH_D_LOCK(expand_tag, 0)
 #undef expand_tag
 } kmp_direct_locktag_t;
 
 // Indirect lock type
 typedef struct {
     kmp_user_lock_p lock;
     kmp_indirect_locktag_t type;
 } kmp_indirect_lock_t;
 
 // Function tables for direct locks. Set/unset/test differentiate functions with/without consistency checking.
-extern void (*__kmp_direct_init_ops[])(kmp_dyna_lock_t *, kmp_dyna_lockseq_t);
-extern void (*__kmp_direct_destroy_ops[])(kmp_dyna_lock_t *);
-extern void (*(*__kmp_direct_set_ops))(kmp_dyna_lock_t *, kmp_int32);
-extern void (*(*__kmp_direct_unset_ops))(kmp_dyna_lock_t *, kmp_int32);
-extern int  (*(*__kmp_direct_test_ops))(kmp_dyna_lock_t *, kmp_int32);
+extern void (*__kmp_direct_init[])(kmp_dyna_lock_t *, kmp_dyna_lockseq_t);
+extern void (*__kmp_direct_destroy[])(kmp_dyna_lock_t *);
+extern void (*(*__kmp_direct_set))(kmp_dyna_lock_t *, kmp_int32);
+extern int  (*(*__kmp_direct_unset))(kmp_dyna_lock_t *, kmp_int32);
+extern int  (*(*__kmp_direct_test))(kmp_dyna_lock_t *, kmp_int32);
 
 // Function tables for indirect locks. Set/unset/test differentiate functions with/withuot consistency checking.
-extern void (*__kmp_indirect_init_ops[])(kmp_user_lock_p);
-extern void (*__kmp_indirect_destroy_ops[])(kmp_user_lock_p);
-extern void (*(*__kmp_indirect_set_ops))(kmp_user_lock_p, kmp_int32);
-extern void (*(*__kmp_indirect_unset_ops))(kmp_user_lock_p, kmp_int32);
-extern int  (*(*__kmp_indirect_test_ops))(kmp_user_lock_p, kmp_int32);
+extern void (*__kmp_indirect_init[])(kmp_user_lock_p);
+extern void (*__kmp_indirect_destroy[])(kmp_user_lock_p);
+extern void (*(*__kmp_indirect_set))(kmp_user_lock_p, kmp_int32);
+extern int  (*(*__kmp_indirect_unset))(kmp_user_lock_p, kmp_int32);
+extern int  (*(*__kmp_indirect_test))(kmp_user_lock_p, kmp_int32);
 
 // Extracts direct lock tag from a user lock pointer
-#define KMP_EXTRACT_D_TAG(l)   (*((kmp_dyna_lock_t *)(l)) & KMP_LOCK_TYPE_MASK & -(*((kmp_dyna_lock_t *)(l)) & 1))
+#define KMP_EXTRACT_D_TAG(l)   (*((kmp_dyna_lock_t *)(l)) & ((1<<KMP_LOCK_SHIFT)-1) & -(*((kmp_dyna_lock_t *)(l)) & 1))
 
 // Extracts indirect lock index from a user lock pointer
 #define KMP_EXTRACT_I_INDEX(l) (*(kmp_lock_index_t *)(l) >> 1)
 
 // Returns function pointer to the direct lock function with l (kmp_dyna_lock_t *) and op (operation type).
-#define KMP_D_LOCK_FUNC(l, op) __kmp_direct_##op##_ops[KMP_EXTRACT_D_TAG(l)]
+#define KMP_D_LOCK_FUNC(l, op) __kmp_direct_##op[KMP_EXTRACT_D_TAG(l)]
 
 // Returns function pointer to the indirect lock function with l (kmp_indirect_lock_t *) and op (operation type).
-#define KMP_I_LOCK_FUNC(l, op) __kmp_indirect_##op##_ops[((kmp_indirect_lock_t *)(l))->type]
+#define KMP_I_LOCK_FUNC(l, op) __kmp_indirect_##op[((kmp_indirect_lock_t *)(l))->type]
 
 // Initializes a direct lock with the given lock pointer and lock sequence.
-#define KMP_INIT_D_LOCK(l, seq) __kmp_direct_init_ops[KMP_GET_D_TAG(seq)]((kmp_dyna_lock_t *)l, seq)
+#define KMP_INIT_D_LOCK(l, seq) __kmp_direct_init[KMP_GET_D_TAG(seq)]((kmp_dyna_lock_t *)l, seq)
 
 // Initializes an indirect lock with the given lock pointer and lock sequence.
-#define KMP_INIT_I_LOCK(l, seq) __kmp_direct_init_ops[0]((kmp_dyna_lock_t *)(l), seq)
+#define KMP_INIT_I_LOCK(l, seq) __kmp_direct_init[0]((kmp_dyna_lock_t *)(l), seq)
 
 // Returns "free" lock value for the given lock type.
 #define KMP_LOCK_FREE(type)      (locktag_##type)
 
 // Returns "busy" lock value for the given lock teyp.
-#define KMP_LOCK_BUSY(v, type)   ((v)<<KMP_LOCK_VALUE_SHIFT | locktag_##type)
+#define KMP_LOCK_BUSY(v, type)   ((v)<<KMP_LOCK_SHIFT | locktag_##type)
 
 // Returns lock value after removing (shifting) lock tag.
-#define KMP_LOCK_STRIP(v)        ((v)>>KMP_LOCK_VALUE_SHIFT)
+#define KMP_LOCK_STRIP(v)        ((v)>>KMP_LOCK_SHIFT)
 
 // Updates __kmp_user_lock_seq with the give lock type.
 #define KMP_STORE_LOCK_SEQ(type) (__kmp_user_lock_seq = lockseq_##type)
 
 // Internal entries for hinted lock initializers.
 extern void __kmp_init_lock_hinted(void **, int);
 extern void __kmp_init_nest_lock_hinted(void **, int);
 

openmp/trunk/runtime/src/kmp_lock.cpp

 }
 
 #if KMP_USE_ADAPTIVE_LOCKS
 
 /*
     RTM Adaptive locks
 */
 
-// TODO: Use the header for intrinsics below with the compiler 13.0
-//#include <immintrin.h>
+#if KMP_COMPILER_ICC && __INTEL_COMPILER >= 1300
+
+#include <immintrin.h>
+#define SOFT_ABORT_MASK  (_XABORT_RETRY | _XABORT_CONFLICT | _XABORT_EXPLICIT)
+
+#else
 
 // Values from the status register after failed speculation.
 #define _XBEGIN_STARTED          (~0u)
 #define _XABORT_EXPLICIT         (1 << 0)
 #define _XABORT_RETRY            (1 << 1)
 #define _XABORT_CONFLICT         (1 << 2)
 #define _XABORT_CAPACITY         (1 << 3)
 #define _XABORT_DEBUG            (1 << 4)
#define _xabort(ARG)                            \
     _asm _emit 0xc6                             \
     _asm _emit 0xf8                             \
     _asm _emit ARG
 #else
 #define _xabort(ARG) \
     __asm__ volatile (".byte 0xC6; .byte 0xF8; .byte " STRINGIZE(ARG) :::"memory");
 #endif
 
+#endif // KMP_COMPILER_ICC && __INTEL_COMPILER >= 1300
+
 //
 //    Statistics is collected for testing purpose
 //
 #if KMP_DEBUG_ADAPTIVE_LOCKS
 
 // We accumulate speculative lock statistics when the lock is destroyed.
 // We keep locks that haven't been destroyed in the liveLocks list
 // so that we can grab their statistics too.
do
            * and now. This also gets the lock cache line into our read-set,
            * which we need so that we'll abort if anyone later claims it for real.
            */
             if (! __kmp_is_unlocked_queuing_lock( GET_QLK_PTR(lck) ) )
             {
                 // Lock is now visibly acquired, so someone beat us to it.
                 // Abort the transaction so we'll restart from _xbegin with the
                 // failure status.
-                _xabort(0x01)
+                _xabort(0x01);
                 KMP_ASSERT2( 0, "should not get here" );
             }
             return 1;   // Lock has been acquired (speculatively)
         } else {
             // We have aborted, update the statistics
             if ( status & SOFT_ABORT_MASK)
             {
                 KMP_INC_STAT(lck,softFailedSpeculations);

 static void
 __kmp_set_drdpa_lock_flags( kmp_drdpa_lock_t *lck, kmp_lock_flags_t flags )
 {
     lck->lk.flags = flags;
 }
 
 #if KMP_USE_DYNAMIC_LOCK
 
-// Definitions of lock hints.
+// Definitions of lock hints - can't include omp.h because of other name clashes.
 # ifndef __OMP_H 
 typedef enum kmp_lock_hint_t {
     kmp_lock_hint_none = 0,
     kmp_lock_hint_contended,
     kmp_lock_hint_uncontended,
     kmp_lock_hint_nonspeculative,
     kmp_lock_hint_speculative,
     kmp_lock_hint_adaptive,
 } kmp_lock_hint_t;
 # endif
 
 // Direct lock initializers. It simply writes a tag to the low 8 bits of the lock word.
-#define expand_init_lock(l, a)                                              \
-static void init_##l##_lock(kmp_dyna_lock_t *lck, kmp_dyna_lockseq_t seq) { \
-    *lck = KMP_LOCK_FREE(l);                                                \
-    KA_TRACE(20, ("Initialized direct lock, tag = %x\n", *lck));            \
+static void __kmp_init_direct_lock(kmp_dyna_lock_t *lck, kmp_dyna_lockseq_t seq)
+{
+    TCW_4(*lck, KMP_GET_D_TAG(seq));
+    KA_TRACE(20, ("__kmp_init_direct_lock: initialized direct lock with type#%d\n", seq));
 }
-FOREACH_D_LOCK(expand_init_lock, 0)
-#undef expand_init_lock
 
 #if KMP_HAS_HLE
 
 // HLE lock functions - imported from the testbed runtime.
-#if KMP_MIC
-# define machine_pause() _mm_delay_32(10) // TODO: find the right argument
-#else
-# define machine_pause() _mm_pause()
-#endif
 #define HLE_ACQUIRE ".byte 0xf2;"
 #define HLE_RELEASE ".byte 0xf3;"
 
 static inline kmp_uint32
 swap4(kmp_uint32 volatile *p, kmp_uint32 v)
 {
     __asm__ volatile(HLE_ACQUIRE "xchg %1,%0"
                     : "+r"(v), "+m"(*p)
                     :
                     : "memory");
     return v;
 }
 
 static void
 __kmp_destroy_hle_lock(kmp_dyna_lock_t *lck)
 {
-    *lck = 0;
+    TCW_4(*lck, 0);
 }
 
 static void
 __kmp_acquire_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid)
 {
     // Use gtid for KMP_LOCK_BUSY if necessary
     if (swap4(lck, KMP_LOCK_BUSY(1, hle)) != KMP_LOCK_FREE(hle)) {
         int delay = 1;
         do {
             while (*(kmp_uint32 volatile *)lck != KMP_LOCK_FREE(hle)) {
                 for (int i = delay; i != 0; --i)
-                    machine_pause();
+                    KMP_CPU_PAUSE();
                 delay = ((delay << 1) | 1) & 7;
             }
         } while (swap4(lck, KMP_LOCK_BUSY(1, hle)) != KMP_LOCK_FREE(hle));
     }
 }
 
 static void
 __kmp_acquire_hle_lock_with_checks(kmp_dyna_lock_t *lck, kmp_int32 gtid)
 {
     __kmp_acquire_hle_lock(lck, gtid); // TODO: add checks
 }
 
-static void
+static int
 __kmp_release_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid)
 {
     __asm__ volatile(HLE_RELEASE "movl %1,%0"
                     : "=m"(*lck)
                     : "r"(KMP_LOCK_FREE(hle))
                     : "memory");
+    return KMP_LOCK_RELEASED;
 }
 
-static void
+static int
 __kmp_release_hle_lock_with_checks(kmp_dyna_lock_t *lck, kmp_int32 gtid)
 {
-    __kmp_release_hle_lock(lck, gtid); // TODO: add checks
+    return __kmp_release_hle_lock(lck, gtid); // TODO: add checks
 }
 
 static int
 __kmp_test_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid)
 {
     return swap4(lck, KMP_LOCK_BUSY(1, hle)) == KMP_LOCK_FREE(hle);
 }
 
 static int
 __kmp_test_hle_lock_with_checks(kmp_dyna_lock_t *lck, kmp_int32 gtid)
 {
     return __kmp_test_hle_lock(lck, gtid); // TODO: add checks
 }
 
 #endif // KMP_HAS_HLE
 
 // Entry functions for indirect locks (first element of direct_*_ops[]).
 static void __kmp_init_indirect_lock(kmp_dyna_lock_t * l, kmp_dyna_lockseq_t tag);
 static void __kmp_destroy_indirect_lock(kmp_dyna_lock_t * lock);
 static void __kmp_set_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32);
-static void __kmp_unset_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32);
+static int  __kmp_unset_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32);
 static int  __kmp_test_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32);
 static void __kmp_set_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32);
-static void __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32);
+static int  __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32);
 static int  __kmp_test_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32);
 
 //
 // Jump tables for the indirect lock functions.
 // Only fill in the odd entries, that avoids the need to shift out the low bit.
 //
-#define expand_func0(l, op) 0,op##_##l##_##lock,
-void (*__kmp_direct_init_ops[])(kmp_dyna_lock_t *, kmp_dyna_lockseq_t)
-    = { __kmp_init_indirect_lock, 0, FOREACH_D_LOCK(expand_func0, init) };
-
-#define expand_func1(l, op) 0,(void (*)(kmp_dyna_lock_t *))__kmp_##op##_##l##_##lock,
-void (*__kmp_direct_destroy_ops[])(kmp_dyna_lock_t *)
-    = { __kmp_destroy_indirect_lock, 0, FOREACH_D_LOCK(expand_func1, destroy) };
-
-// Differentiates *lock and *lock_with_checks.
-#define expand_func2(l, op)  0,(void (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_##lock,
-#define expand_func2c(l, op) 0,(void (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_##lock_with_checks,
-static void (*direct_set_tab[][KMP_NUM_D_LOCKS*2+2])(kmp_dyna_lock_t *, kmp_int32)
-    = { { __kmp_set_indirect_lock, 0, FOREACH_D_LOCK(expand_func2, acquire)  },
-        { __kmp_set_indirect_lock_with_checks, 0, FOREACH_D_LOCK(expand_func2c, acquire) } };
-static void (*direct_unset_tab[][KMP_NUM_D_LOCKS*2+2])(kmp_dyna_lock_t *, kmp_int32)
-    = { { __kmp_unset_indirect_lock, 0, FOREACH_D_LOCK(expand_func2, release)  },
-        { __kmp_unset_indirect_lock_with_checks, 0, FOREACH_D_LOCK(expand_func2c, release) } };
-
-#define expand_func3(l, op)  0,(int  (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_##lock,
-#define expand_func3c(l, op) 0,(int  (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_##lock_with_checks,
-static int  (*direct_test_tab[][KMP_NUM_D_LOCKS*2+2])(kmp_dyna_lock_t *, kmp_int32)
-    = { { __kmp_test_indirect_lock, 0, FOREACH_D_LOCK(expand_func3, test)  },
-        { __kmp_test_indirect_lock_with_checks, 0, FOREACH_D_LOCK(expand_func3c, test) } };
+
+// init functions
+#define expand(l, op) 0,__kmp_init_direct_lock,
+void (*__kmp_direct_init[])(kmp_dyna_lock_t *, kmp_dyna_lockseq_t)
+    = { __kmp_init_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, init) };
+#undef expand
+
+// destroy functions
+#define expand(l, op) 0,(void (*)(kmp_dyna_lock_t *))__kmp_##op##_##l##_lock,
+void (*__kmp_direct_destroy[])(kmp_dyna_lock_t *)
+    = { __kmp_destroy_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, destroy) };
+#undef expand
+
+// set/acquire functions
+#define expand(l, op) 0,(void (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock,
+static void (*direct_set[])(kmp_dyna_lock_t *, kmp_int32)
+    = { __kmp_set_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, acquire) };
+#undef expand
+#define expand(l, op) 0,(void (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock_with_checks,
+static void (*direct_set_check[])(kmp_dyna_lock_t *, kmp_int32)
+    = { __kmp_set_indirect_lock_with_checks, 0, KMP_FOREACH_D_LOCK(expand, acquire) };
+#undef expand
+
+// unset/release and test functions
+#define expand(l, op) 0,(int  (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock,
+static int (*direct_unset[])(kmp_dyna_lock_t *, kmp_int32)
+    = { __kmp_unset_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, release) };
+static int  (*direct_test[])(kmp_dyna_lock_t *, kmp_int32)
+    = { __kmp_test_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, test) };
+#undef expand
+#define expand(l, op) 0,(int  (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock_with_checks,
+static int (*direct_unset_check[])(kmp_dyna_lock_t *, kmp_int32)
+    = { __kmp_unset_indirect_lock_with_checks, 0, KMP_FOREACH_D_LOCK(expand, release) };
+static int (*direct_test_check[])(kmp_dyna_lock_t *, kmp_int32)
+    = { __kmp_test_indirect_lock_with_checks, 0, KMP_FOREACH_D_LOCK(expand, test) };
+#undef expand
 
 // Exposes only one set of jump tables (*lock or *lock_with_checks).
-void (*(*__kmp_direct_set_ops))(kmp_dyna_lock_t *, kmp_int32) = 0;
-void (*(*__kmp_direct_unset_ops))(kmp_dyna_lock_t *, kmp_int32) = 0;
-int (*(*__kmp_direct_test_ops))(kmp_dyna_lock_t *, kmp_int32) = 0;
+void (*(*__kmp_direct_set))(kmp_dyna_lock_t *, kmp_int32) = 0;
+int (*(*__kmp_direct_unset))(kmp_dyna_lock_t *, kmp_int32) = 0;
+int (*(*__kmp_direct_test))(kmp_dyna_lock_t *, kmp_int32) = 0;
 
 //
 // Jump tables for the indirect lock functions.
 //
-#define expand_func4(l, op) (void (*)(kmp_user_lock_p))__kmp_##op##_##l##_##lock,
-void (*__kmp_indirect_init_ops[])(kmp_user_lock_p)
-    = { FOREACH_I_LOCK(expand_func4, init) };
-void (*__kmp_indirect_destroy_ops[])(kmp_user_lock_p)
-    = { FOREACH_I_LOCK(expand_func4, destroy) };
-
-// Differentiates *lock and *lock_with_checks.
-#define expand_func5(l, op)  (void (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock,
-#define expand_func5c(l, op) (void (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock_with_checks,
-static void (*indirect_set_tab[][KMP_NUM_I_LOCKS])(kmp_user_lock_p, kmp_int32)
-    = { { FOREACH_I_LOCK(expand_func5, acquire)  },
-        { FOREACH_I_LOCK(expand_func5c, acquire) } };
-static void (*indirect_unset_tab[][KMP_NUM_I_LOCKS])(kmp_user_lock_p, kmp_int32)
-    = { { FOREACH_I_LOCK(expand_func5, release)  },
-        { FOREACH_I_LOCK(expand_func5c, release) } };
-
-#define expand_func6(l, op)  (int  (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock,
-#define expand_func6c(l, op) (int  (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock_with_checks,
-static int  (*indirect_test_tab[][KMP_NUM_I_LOCKS])(kmp_user_lock_p, kmp_int32)
-    = { { FOREACH_I_LOCK(expand_func6, test)  },
-        { FOREACH_I_LOCK(expand_func6c, test) } };
-
-// Exposes only one set of jump tables (*lock or *lock_with_checks).
-void (*(*__kmp_indirect_set_ops))(kmp_user_lock_p, kmp_int32) = 0;
-void (*(*__kmp_indirect_unset_ops))(kmp_user_lock_p, kmp_int32) = 0;
-int (*(*__kmp_indirect_test_ops))(kmp_user_lock_p, kmp_int32) = 0;
+#define expand(l, op) (void (*)(kmp_user_lock_p))__kmp_##op##_##l##_##lock,
+void (*__kmp_indirect_init[])(kmp_user_lock_p) = { KMP_FOREACH_I_LOCK(expand, init) };
+void (*__kmp_indirect_destroy[])(kmp_user_lock_p) = { KMP_FOREACH_I_LOCK(expand, destroy) };
+#undef expand
+
+// set/acquire functions
+#define expand(l, op) (void (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock,
+static void (*indirect_set[])(kmp_user_lock_p, kmp_int32) = { KMP_FOREACH_I_LOCK(expand, acquire) };
+#undef expand
+#define expand(l, op) (void (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock_with_checks,
+static void (*indirect_set_check[])(kmp_user_lock_p, kmp_int32) = { KMP_FOREACH_I_LOCK(expand, acquire) };
+#undef expand
+
+// unset/release and test functions
+#define expand(l, op) (int  (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock,
+static int (*indirect_unset[])(kmp_user_lock_p, kmp_int32) = { KMP_FOREACH_I_LOCK(expand, release) };
+static int (*indirect_test[])(kmp_user_lock_p, kmp_int32) = { KMP_FOREACH_I_LOCK(expand, test) };
+#undef expand
+#define expand(l, op) (int  (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock_with_checks,
+static int (*indirect_unset_check[])(kmp_user_lock_p, kmp_int32) = { KMP_FOREACH_I_LOCK(expand, release) };
+static int (*indirect_test_check[])(kmp_user_lock_p, kmp_int32) = { KMP_FOREACH_I_LOCK(expand, test) };
+#undef expand
+
+// Exposes only one jump tables (*lock or *lock_with_checks).
+void (*(*__kmp_indirect_set))(kmp_user_lock_p, kmp_int32) = 0;
+int (*(*__kmp_indirect_unset))(kmp_user_lock_p, kmp_int32) = 0;
+int (*(*__kmp_indirect_test))(kmp_user_lock_p, kmp_int32) = 0;
 
 // Lock index table.
 kmp_indirect_lock_t **__kmp_indirect_lock_table;
 kmp_lock_index_t __kmp_indirect_lock_table_size;
 kmp_lock_index_t __kmp_indirect_lock_table_next;
 
 // Size of indirect locks.
 static kmp_uint32 __kmp_indirect_lock_size[KMP_NUM_I_LOCKS] = {

 
 static void
 __kmp_set_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32 gtid)
 {
     kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
     KMP_I_LOCK_FUNC(l, set)(l->lock, gtid);
 }
 
-static void
+static int
 __kmp_unset_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32 gtid)
 {
     kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
-    KMP_I_LOCK_FUNC(l, unset)(l->lock, gtid);
+    return KMP_I_LOCK_FUNC(l, unset)(l->lock, gtid);
 }
 
 static int
 __kmp_test_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32 gtid)
 {
     kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
     return KMP_I_LOCK_FUNC(l, test)(l->lock, gtid);
 }
 
 static void
 __kmp_set_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32 gtid)
 {
     kmp_indirect_lock_t *l = __kmp_lookup_indirect_lock((void **)lock, "omp_set_lock");
     KMP_I_LOCK_FUNC(l, set)(l->lock, gtid);
 }
 
-static void
+static int
 __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32 gtid)
 {
     kmp_indirect_lock_t *l = __kmp_lookup_indirect_lock((void **)lock, "omp_unset_lock");
-    KMP_I_LOCK_FUNC(l, unset)(l->lock, gtid);
+    return KMP_I_LOCK_FUNC(l, unset)(l->lock, gtid);
 }
 
 static int
 __kmp_test_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32 gtid)
 {
     kmp_indirect_lock_t *l = __kmp_lookup_indirect_lock((void **)lock, "omp_test_lock");
     return KMP_I_LOCK_FUNC(l, test)(l->lock, gtid);
 }
__kmp_init_nest_lock_hinted(void **lock, int hint)
     }
     KMP_INIT_I_LOCK(lock, seq);
 #if USE_ITT_BUILD
     kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(lock);
     __kmp_itt_lock_creating(ilk->lock, NULL);
 #endif
 }
 
-// Initializes the lock table for indirect locks.
-static void
-__kmp_init_indirect_lock_table()
-{
-    __kmp_indirect_lock_table = (kmp_indirect_lock_t **)__kmp_allocate(sizeof(kmp_indirect_lock_t *)*1024);
-    __kmp_indirect_lock_table_size = 1024;
-    __kmp_indirect_lock_table_next = 0;
-}
-
 #if KMP_USE_ADAPTIVE_LOCKS
 # define init_lock_func(table, expand) {             \
     table[locktag_ticket]         = expand(ticket);  \
     table[locktag_queuing]        = expand(queuing); \
     table[locktag_adaptive]       = expand(queuing); \
     table[locktag_drdpa]          = expand(drdpa);   \
     table[locktag_nested_ticket]  = expand(ticket);  \
     table[locktag_nested_queuing] = expand(queuing); \
# define init_lock_func(table, expand) {             \
     table[locktag_nested_drdpa]   = expand(drdpa);   \
 }
 #endif // KMP_USE_ADAPTIVE_LOCKS
 
 // Initializes data for dynamic user locks.
 void
 __kmp_init_dynamic_user_locks()
 {
-    // Initialize jump table location
-    int offset = (__kmp_env_consistency_check)? 1: 0;
-    __kmp_direct_set_ops = direct_set_tab[offset];
-    __kmp_direct_unset_ops = direct_unset_tab[offset];
-    __kmp_direct_test_ops = direct_test_tab[offset];
-    __kmp_indirect_set_ops = indirect_set_tab[offset];
-    __kmp_indirect_unset_ops = indirect_unset_tab[offset];
-    __kmp_indirect_test_ops = indirect_test_tab[offset];
-    __kmp_init_indirect_lock_table();
+    // Initialize jump table for the lock functions
+    if (__kmp_env_consistency_check) {
+        __kmp_direct_set     = direct_set_check;
+        __kmp_direct_unset   = direct_unset_check;
+        __kmp_direct_test    = direct_test_check;
+        __kmp_indirect_set   = indirect_set_check;
+        __kmp_indirect_unset = indirect_unset_check;
+        __kmp_indirect_test  = indirect_test_check;
+    }
+    else {
+        __kmp_direct_set     = direct_set;
+        __kmp_direct_unset   = direct_unset;
+        __kmp_direct_test    = direct_test;
+        __kmp_indirect_set   = indirect_set;
+        __kmp_indirect_unset = indirect_unset;
+        __kmp_indirect_test  = indirect_test;
+    }
+
+    // Initialize lock index table
+    __kmp_indirect_lock_table = (kmp_indirect_lock_t **)__kmp_allocate(sizeof(kmp_indirect_lock_t *)*1024);
+    __kmp_indirect_lock_table_size = 1024;
+    __kmp_indirect_lock_table_next = 0;
 
     // Initialize lock accessor/modifier
     // Could have used designated initializer, but -TP /Qstd=c99 did not work with icl.exe.
 #define expand_func(l) (void (*)(kmp_user_lock_p, const ident_t *))__kmp_set_##l##_lock_location
     init_lock_func(__kmp_indirect_set_location, expand_func);
 #undef expand_func
 #define expand_func(l) (void (*)(kmp_user_lock_p, kmp_lock_flags_t))__kmp_set_##l##_lock_flags
     init_lock_func(__kmp_indirect_set_flags, expand_func);