[OpenMP] Remove TSan annotations from libomp

[protze.joachim]

Summary

The annotations in libomp were never built by default. The annotations are also superseded by the OMPT tool libarcher.so, which is automatically loaded if an OpenMP application is built with TSan.
This patch removes all TSan annotations from the runtime code.

[ye-luo]

"libarcher.so, which is automatically loaded if an OpenMP application is built with TSan." Is this valid with gcc + libomp combination?

[protze.joachim]

In D103767#2801322, @ye-luo wrote:

"libarcher.so, which is automatically loaded if an OpenMP application is built with TSan." Is this valid with gcc + libomp combination?

Yes. The OMPT startup code in libomp assumes libarcher.so to be the last element of the OMP_TOOL_LIBARARIES var. So, as long as you don't load a different OMPT tool while executing an application built with TSan, you will have Archer loaded. To verify, you can export ARCHER_OPTIONS=verbose=1. If you have doubts or if loading Archer fails for you, you can export OMP_TOOL_VERBOSE_INIT=stdout to get detailed information about OMPT initialization including dlerror output.

[protze.joachim]

@AndreyChurbanov we talked about the patch during today's OpenMP in LLVM call and agreed that the patch should be applied. Can you please review and accept?

I just removed one unrelated change and rebased to current main.

[JonChesterfield]

Is archer a functionally equivalent replacement for TSAN?

I'm surprised that it can detect races without annotation where tsan does require annotations. There isn't anything about openmp that makes that obviously possible, so is it a case of features that could be added to tsan, such that we could run tsan against this without the annotations?

[protze.joachim]

Probably, my statement was misleading. Archer is effectively an adapter between the OpenMP tools interface (OMPT) and the Annotation interface in TSan.
Archer does not perform its own data race analysis, but translates OpenMP synchronization semantics (derived from OMPT event callbacks) into TSan annotations.

Since libarcher gets loaded by default, enabling LIBOMP_TSAN_SUPPORT will result in redundant annotations for TSan, which slightly differ in details and coverage.

[AndreyChurbanov]

LGTM

openmp/runtime/CMakeLists.txt

 set(LIBOMP_OMPT_DEBUG FALSE CACHE BOOL
   "Trace OMPT initialization?")
 set(LIBOMP_OMPT_OPTIONAL TRUE CACHE BOOL
   "OMPT-optional?")
 if(LIBOMP_OMPT_SUPPORT AND (NOT LIBOMP_HAVE_OMPT_SUPPORT))
   libomp_error_say("OpenMP Tools Interface requested but not available in this implementation")
 endif()
 
-# TSAN-support
-set(LIBOMP_TSAN_SUPPORT FALSE CACHE BOOL
-  "TSAN-support?")
-if(LIBOMP_TSAN_SUPPORT AND (NOT LIBOMP_HAVE_TSAN_SUPPORT))
-  libomp_error_say("TSAN functionality requested but not available")
-endif()
-
 # OMPD-support
 # Enable if OMPT SUPPORT is ON
 set(OMPD_DEFAULT FALSE)
 if (LIBOMP_HAVE_OMPT_SUPPORT AND ("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux"))
   set(OMPD_DEFAULT TRUE)
 endif()
 
 set(LIBOMP_OMPD_SUPPORT ${OMPD_DEFAULT} CACHE BOOL
if(${OPENMP_STANDALONE_BUILD})
   libomp_say("Use ITT notify       -- ${LIBOMP_USE_ITT_NOTIFY}")
   libomp_say("Use OMPT-support     -- ${LIBOMP_OMPT_SUPPORT}")
   if(${LIBOMP_OMPT_SUPPORT})
     libomp_say("Use OMPT-optional  -- ${LIBOMP_OMPT_OPTIONAL}")
   endif()
   libomp_say("Use OMPD-support     -- ${LIBOMP_OMPD_SUPPORT}")
   libomp_say("Use Adaptive locks   -- ${LIBOMP_USE_ADAPTIVE_LOCKS}")
   libomp_say("Use quad precision   -- ${LIBOMP_USE_QUAD_PRECISION}")
-  libomp_say("Use TSAN-support     -- ${LIBOMP_TSAN_SUPPORT}")
   libomp_say("Use Hwloc library    -- ${LIBOMP_USE_HWLOC}")
 endif()
 
 add_subdirectory(src)
 add_subdirectory(test)
 
 # make these variables available for tools:
 set(LIBOMP_LIBRARY_DIR ${LIBOMP_LIBRARY_DIR} PARENT_SCOPE)
 set(LIBOMP_INCLUDE_DIR ${LIBOMP_INCLUDE_DIR} PARENT_SCOPE)
 # make these variables available for tools/libompd:
 set(LIBOMP_SRC_DIR ${LIBOMP_SRC_DIR} PARENT_SCOPE)

openmp/runtime/src/CMakeLists.txt

else()
   libomp_append(LIBOMP_CXXFILES kmp_taskdeps.cpp)
   libomp_append(LIBOMP_CXXFILES kmp_cancel.cpp)
 endif()
 # Files common to stubs and normal library
 libomp_append(LIBOMP_CXXFILES kmp_ftn_cdecl.cpp)
 libomp_append(LIBOMP_CXXFILES kmp_ftn_extra.cpp)
 libomp_append(LIBOMP_CXXFILES kmp_version.cpp)
 libomp_append(LIBOMP_CXXFILES ompt-general.cpp IF_TRUE LIBOMP_OMPT_SUPPORT)
-libomp_append(LIBOMP_CXXFILES tsan_annotations.cpp IF_TRUE LIBOMP_TSAN_SUPPORT)
 libomp_append(LIBOMP_CXXFILES ompd-specific.cpp IF_TRUE LIBOMP_OMPD_SUPPORT)
 
 set(LIBOMP_SOURCE_FILES ${LIBOMP_CXXFILES} ${LIBOMP_ASMFILES})
 # For Windows, there is a resource file (.rc -> .res) that is also compiled
 libomp_append(LIBOMP_SOURCE_FILES libomp.rc WIN32)
 
 # Get compiler and assembler flags
 libomp_get_cxxflags(LIBOMP_CONFIGURED_CXXFLAGS)

openmp/runtime/src/exports_so.txt

global: # Exported symbols.
         omp_*;     # Standard OpenMP functions.
         OMP_*;     # Standard OpenMP symbols.
 
         #
         # OMPT API
         #
         ompt_start_tool;     # OMPT start interface
 
-        # icc drops weak attribute at linking step without the following line:
-        Annotate*;           # TSAN annotation
-
         ompc_*;    # omp.h renames some standard functions to ompc_*.
         kmp_*;     # Intel extensions.
         kmpc_*;    # Intel extensions.
         __kmpc_*;  # Functions called by compiler-generated code.
         GOMP_*;    # GNU C compatibility functions.
 
         _You_must_link_with_*;     # Mutual detection/MS compatibility symbols.
 

openmp/runtime/src/kmp_barrier.cpp

 #include "kmp_stats.h"
 #include "ompt-specific.h"
 
 #if KMP_MIC
 #include <immintrin.h>
 #define USE_NGO_STORES 1
 #endif // KMP_MIC
 
-#include "tsan_annotations.h"
-
 #if KMP_MIC && USE_NGO_STORES
 // ICV copying
 #define ngo_load(src) __m512d Vt = _mm512_load_pd((void *)(src))
 #define ngo_store_icvs(dst, src) _mm512_storenrngo_pd((void *)(dst), Vt)
 #define ngo_store_go(dst, src) _mm512_storenrngo_pd((void *)(dst), Vt)
 #define ngo_sync() __asm__ volatile("lock; addl $0,0(%%rsp)" ::: "memory")
 #else
 #define ngo_load(src) ((void)0)
KA_TRACE(20,
               "arrived(%p): %llu => %llu\n",
               gtid, team->t.t_id, tid, __kmp_gtid_from_tid(0, team),
               team->t.t_id, 0, &thr_bar->b_arrived, thr_bar->b_arrived,
               thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
     // Mark arrival to primary thread
     /* After performing this write, a worker thread may not assume that the team
        is valid any more - it could be deallocated by the primary thread at any
        time. */
-    ANNOTATE_BARRIER_BEGIN(this_thr);
     kmp_flag_64<> flag(&thr_bar->b_arrived, other_threads[0]);
     flag.release();
   } else {
     kmp_balign_team_t *team_bar = &team->t.t_bar[bt];
     int nproc = this_thr->th.th_team_nproc;
     int i;
     // Don't have to worry about sleep bit here or atomic since team setting
     kmp_uint64 new_state = team_bar->b_arrived + KMP_BARRIER_STATE_BUMP;
#endif /* KMP_CACHE_MANAGE */
             &other_threads[i]->th.th_bar[bt].bb.b_arrived, new_state);
         if (flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj)))
           return true;
       } else {
         kmp_flag_64<> flag(&other_threads[i]->th.th_bar[bt].bb.b_arrived,
                            new_state);
         flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
       }
-      ANNOTATE_BARRIER_END(other_threads[i]);
 #if USE_ITT_BUILD && USE_ITT_NOTIFY
       // Barrier imbalance - write min of the thread time and the other thread
       // time to the thread.
       if (__kmp_forkjoin_frames_mode == 2) {
         this_thr->th.th_bar_min_time = KMP_MIN(
             this_thr->th.th_bar_min_time, other_threads[i]->th.th_bar_min_time);
       }
 #endif
       if (reduce) {
         KA_TRACE(100,
                  ("__kmp_linear_barrier_gather: T#%d(%d:%d) += T#%d(%d:%d)\n",
                   gtid, team->t.t_id, tid, __kmp_gtid_from_tid(i, team),
                   team->t.t_id, i));
-        ANNOTATE_REDUCE_AFTER(reduce);
         OMPT_REDUCTION_DECL(this_thr, gtid);
         OMPT_REDUCTION_BEGIN;
         (*reduce)(this_thr->th.th_local.reduce_data,
                   other_threads[i]->th.th_local.reduce_data);
         OMPT_REDUCTION_END;
-        ANNOTATE_REDUCE_BEFORE(reduce);
-        ANNOTATE_REDUCE_BEFORE(&team->t.t_bar);
       }
     }
     // Don't have to worry about sleep bit here or atomic since team setting
     team_bar->b_arrived = new_state;
     KA_TRACE(20, ("__kmp_linear_barrier_gather: T#%d(%d:%d) set team %d "
                   "arrived(%p) = %llu\n",
                   gtid, team->t.t_id, tid, team->t.t_id, &team_bar->b_arrived,
                   new_state));
#endif /* KMP_CACHE_MANAGE */
         KA_TRACE(
             20,
             ("__kmp_linear_barrier_release: T#%d(%d:%d) releasing T#%d(%d:%d) "
              "go(%p): %u => %u\n",
              gtid, team->t.t_id, tid, other_threads[i]->th.th_info.ds.ds_gtid,
              team->t.t_id, i, &other_threads[i]->th.th_bar[bt].bb.b_go,
              other_threads[i]->th.th_bar[bt].bb.b_go,
              other_threads[i]->th.th_bar[bt].bb.b_go + KMP_BARRIER_STATE_BUMP));
-        ANNOTATE_BARRIER_BEGIN(other_threads[i]);
         kmp_flag_64<> flag(&other_threads[i]->th.th_bar[bt].bb.b_go,
                            other_threads[i]);
         flag.release();
       }
     }
   } else { // Wait for the PRIMARY thread to release us
     KA_TRACE(20, ("__kmp_linear_barrier_release: T#%d wait go(%p) == %u\n",
                   gtid, &thr_bar->b_go, KMP_BARRIER_STATE_BUMP));
     if (cancellable) {
       kmp_flag_64<true, false> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
       if (flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj)))
         return true;
     } else {
       kmp_flag_64<> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
       flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj));
     }
-    ANNOTATE_BARRIER_END(this_thr);
 #if USE_ITT_BUILD && USE_ITT_NOTIFY
     if ((__itt_sync_create_ptr && itt_sync_obj == NULL) || KMP_ITT_DEBUG) {
       // In a fork barrier; cannot get the object reliably (or ITTNOTIFY is
       // disabled)
       itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier, 0, -1);
       // Cancel wait on previous parallel region...
       __kmp_itt_task_starting(itt_sync_obj);
 
#endif /* KMP_CACHE_MANAGE */
       KA_TRACE(20,
                ("__kmp_tree_barrier_gather: T#%d(%d:%d) wait T#%d(%d:%u) "
                 "arrived(%p) == %llu\n",
                 gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
                 team->t.t_id, child_tid, &child_bar->b_arrived, new_state));
       // Wait for child to arrive
       kmp_flag_64<> flag(&child_bar->b_arrived, new_state);
       flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
-      ANNOTATE_BARRIER_END(child_thr);
 #if USE_ITT_BUILD && USE_ITT_NOTIFY
       // Barrier imbalance - write min of the thread time and a child time to
       // the thread.
       if (__kmp_forkjoin_frames_mode == 2) {
         this_thr->th.th_bar_min_time = KMP_MIN(this_thr->th.th_bar_min_time,
                                                child_thr->th.th_bar_min_time);
       }
 #endif
       if (reduce) {
         KA_TRACE(100,
                  ("__kmp_tree_barrier_gather: T#%d(%d:%d) += T#%d(%d:%u)\n",
                   gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
                   team->t.t_id, child_tid));
-        ANNOTATE_REDUCE_AFTER(reduce);
         OMPT_REDUCTION_DECL(this_thr, gtid);
         OMPT_REDUCTION_BEGIN;
         (*reduce)(this_thr->th.th_local.reduce_data,
                   child_thr->th.th_local.reduce_data);
         OMPT_REDUCTION_END;
-        ANNOTATE_REDUCE_BEFORE(reduce);
-        ANNOTATE_REDUCE_BEFORE(&team->t.t_bar);
       }
       child++;
       child_tid++;
     } while (child <= branch_factor && child_tid < nproc);
   }
 
   if (!KMP_MASTER_TID(tid)) { // Worker threads
     kmp_int32 parent_tid = (tid - 1) >> branch_bits;
 
     KA_TRACE(20,
              ("__kmp_tree_barrier_gather: T#%d(%d:%d) releasing T#%d(%d:%d) "
               "arrived(%p): %llu => %llu\n",
               gtid, team->t.t_id, tid, __kmp_gtid_from_tid(parent_tid, team),
               team->t.t_id, parent_tid, &thr_bar->b_arrived, thr_bar->b_arrived,
               thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
 
     // Mark arrival to parent thread
     /* After performing this write, a worker thread may not assume that the team
        is valid any more - it could be deallocated by the primary thread at any
        time.  */
-    ANNOTATE_BARRIER_BEGIN(this_thr);
     kmp_flag_64<> flag(&thr_bar->b_arrived, other_threads[parent_tid]);
     flag.release();
   } else {
     // Need to update the team arrived pointer if we are the primary thread
     if (nproc > 1) // New value was already computed above
       team->t.t_bar[bt].b_arrived = new_state;
     else
       team->t.t_bar[bt].b_arrived += KMP_BARRIER_STATE_BUMP;
static void __kmp_tree_barrier_release(
   // Perform a tree release for all of the threads that have been gathered
   if (!KMP_MASTER_TID(
           tid)) { // Handle fork barrier workers who aren't part of a team yet
     KA_TRACE(20, ("__kmp_tree_barrier_release: T#%d wait go(%p) == %u\n", gtid,
                   &thr_bar->b_go, KMP_BARRIER_STATE_BUMP));
     // Wait for parent thread to release us
     kmp_flag_64<> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
     flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj));
-    ANNOTATE_BARRIER_END(this_thr);
 #if USE_ITT_BUILD && USE_ITT_NOTIFY
     if ((__itt_sync_create_ptr && itt_sync_obj == NULL) || KMP_ITT_DEBUG) {
       // In fork barrier where we could not get the object reliably (or
       // ITTNOTIFY is disabled)
       itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier, 0, -1);
       // Cancel wait on previous parallel region...
       __kmp_itt_task_starting(itt_sync_obj);
 

 #endif // KMP_BARRIER_ICV_PUSH
       KA_TRACE(20,
                ("__kmp_tree_barrier_release: T#%d(%d:%d) releasing T#%d(%d:%u)"
                 "go(%p): %u => %u\n",
                 gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
                 team->t.t_id, child_tid, &child_bar->b_go, child_bar->b_go,
                 child_bar->b_go + KMP_BARRIER_STATE_BUMP));
       // Release child from barrier
-      ANNOTATE_BARRIER_BEGIN(child_thr);
       kmp_flag_64<> flag(&child_bar->b_go, child_thr);
       flag.release();
       child++;
       child_tid++;
     } while (child <= branch_factor && child_tid < nproc);
   }
   KA_TRACE(
       20, ("__kmp_tree_barrier_release: T#%d(%d:%d) exit for barrier type %d\n",
if (((tid >> level) & (branch_factor - 1)) != 0) {
                 gtid, team->t.t_id, tid, __kmp_gtid_from_tid(parent_tid, team),
                 team->t.t_id, parent_tid, &thr_bar->b_arrived,
                 thr_bar->b_arrived,
                 thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
       // Mark arrival to parent thread
       /* After performing this write (in the last iteration of the enclosing for
          loop), a worker thread may not assume that the team is valid any more
          - it could be deallocated by the primary thread at any time.  */
-      ANNOTATE_BARRIER_BEGIN(this_thr);
       p_flag.set_waiter(other_threads[parent_tid]);
       p_flag.release();
       break;
     }
 
     // Parent threads wait for children to arrive
     if (new_state == KMP_BARRIER_UNUSED_STATE)
       new_state = team->t.t_bar[bt].b_arrived + KMP_BARRIER_STATE_BUMP;
#endif /* KMP_CACHE_MANAGE */
       KA_TRACE(20,
                ("__kmp_hyper_barrier_gather: T#%d(%d:%d) wait T#%d(%d:%u) "
                 "arrived(%p) == %llu\n",
                 gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
                 team->t.t_id, child_tid, &child_bar->b_arrived, new_state));
       // Wait for child to arrive
       kmp_flag_64<> c_flag(&child_bar->b_arrived, new_state);
       c_flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
-      ANNOTATE_BARRIER_END(child_thr);
       KMP_MB(); // Synchronize parent and child threads.
 #if USE_ITT_BUILD && USE_ITT_NOTIFY
       // Barrier imbalance - write min of the thread time and a child time to
       // the thread.
       if (__kmp_forkjoin_frames_mode == 2) {
         this_thr->th.th_bar_min_time = KMP_MIN(this_thr->th.th_bar_min_time,
                                                child_thr->th.th_bar_min_time);
       }
 #endif
       if (reduce) {
         KA_TRACE(100,
                  ("__kmp_hyper_barrier_gather: T#%d(%d:%d) += T#%d(%d:%u)\n",
                   gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
                   team->t.t_id, child_tid));
-        ANNOTATE_REDUCE_AFTER(reduce);
         OMPT_REDUCTION_DECL(this_thr, gtid);
         OMPT_REDUCTION_BEGIN;
         (*reduce)(this_thr->th.th_local.reduce_data,
                   child_thr->th.th_local.reduce_data);
         OMPT_REDUCTION_END;
-        ANNOTATE_REDUCE_BEFORE(reduce);
-        ANNOTATE_REDUCE_BEFORE(&team->t.t_bar);
       }
     }
   }
 
   if (KMP_MASTER_TID(tid)) {
     // Need to update the team arrived pointer if we are the primary thread
     if (new_state == KMP_BARRIER_UNUSED_STATE)
       team->t.t_bar[bt].b_arrived += KMP_BARRIER_STATE_BUMP;
#if KMP_BARRIER_ICV_PUSH
     }
 #endif
   } else { // Handle fork barrier workers who aren't part of a team yet
     KA_TRACE(20, ("__kmp_hyper_barrier_release: T#%d wait go(%p) == %u\n", gtid,
                   &thr_bar->b_go, KMP_BARRIER_STATE_BUMP));
     // Wait for parent thread to release us
     kmp_flag_64<> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
     flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj));
-    ANNOTATE_BARRIER_END(this_thr);
 #if USE_ITT_BUILD && USE_ITT_NOTIFY
     if ((__itt_sync_create_ptr && itt_sync_obj == NULL) || KMP_ITT_DEBUG) {
       // In fork barrier where we could not get the object reliably
       itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier, 0, -1);
       // Cancel wait on previous parallel region...
       __kmp_itt_task_starting(itt_sync_obj);
 
       if (bt == bs_forkjoin_barrier && TCR_4(__kmp_global.g.g_done))
#endif // KMP_BARRIER_ICV_PUSH
         KA_TRACE(
             20,
             ("__kmp_hyper_barrier_release: T#%d(%d:%d) releasing T#%d(%d:%u)"
              "go(%p): %u => %u\n",
              gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
              team->t.t_id, child_tid, &child_bar->b_go, child_bar->b_go,
              child_bar->b_go + KMP_BARRIER_STATE_BUMP));
         // Release child from barrier
-        ANNOTATE_BARRIER_BEGIN(child_thr);
         kmp_flag_64<> flag(&child_bar->b_go, child_thr);
         flag.release();
       }
     }
   }
 #if KMP_BARRIER_ICV_PUSH
   if (propagate_icvs &&
       !KMP_MASTER_TID(tid)) { // copy ICVs locally to final dest
if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
                 ? thr_bar->b_arrived | thr_bar->leaf_state
                 : team->t.t_bar[bt].b_arrived | thr_bar->leaf_state;
         KA_TRACE(20, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) waiting "
                       "for leaf kids\n",
                       gtid, team->t.t_id, tid));
         kmp_flag_64<> flag(&thr_bar->b_arrived, leaf_state);
         flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
         if (reduce) {
-          ANNOTATE_REDUCE_AFTER(reduce);
           OMPT_REDUCTION_DECL(this_thr, gtid);
           OMPT_REDUCTION_BEGIN;
           for (child_tid = tid + 1; child_tid <= tid + thr_bar->leaf_kids;
                ++child_tid) {
             KA_TRACE(100, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) += "
                            "T#%d(%d:%d)\n",
                            gtid, team->t.t_id, tid,
                            __kmp_gtid_from_tid(child_tid, team), team->t.t_id,
                            child_tid));
-            ANNOTATE_BARRIER_END(other_threads[child_tid]);
             (*reduce)(this_thr->th.th_local.reduce_data,
                       other_threads[child_tid]->th.th_local.reduce_data);
           }
           OMPT_REDUCTION_END;
-          ANNOTATE_REDUCE_BEFORE(reduce);
-          ANNOTATE_REDUCE_BEFORE(&team->t.t_bar);
         }
         // clear leaf_state bits
         KMP_TEST_THEN_AND64(&thr_bar->b_arrived, ~(thr_bar->leaf_state));
       }
       // Next, wait for higher level children on each child's b_arrived flag
       for (kmp_uint32 d = 1; d < thr_bar->my_level;
            ++d) { // gather lowest level threads first, but skip 0
         kmp_uint32 last = tid + thr_bar->skip_per_level[d + 1],
                    skip = thr_bar->skip_per_level[d];
         if (last > nproc)
           last = nproc;
         for (child_tid = tid + skip; child_tid < (int)last; child_tid += skip) {
           kmp_info_t *child_thr = other_threads[child_tid];
           kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb;
           KA_TRACE(20, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) wait "
                         "T#%d(%d:%d) "
                         "arrived(%p) == %llu\n",
                         gtid, team->t.t_id, tid,
                         __kmp_gtid_from_tid(child_tid, team), team->t.t_id,
                         child_tid, &child_bar->b_arrived, new_state));
           kmp_flag_64<> flag(&child_bar->b_arrived, new_state);
           flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
-          ANNOTATE_BARRIER_END(child_thr);
           if (reduce) {
             KA_TRACE(100, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) += "
                            "T#%d(%d:%d)\n",
                            gtid, team->t.t_id, tid,
                            __kmp_gtid_from_tid(child_tid, team), team->t.t_id,
                            child_tid));
-            ANNOTATE_REDUCE_AFTER(reduce);
             (*reduce)(this_thr->th.th_local.reduce_data,
                       child_thr->th.th_local.reduce_data);
-            ANNOTATE_REDUCE_BEFORE(reduce);
-            ANNOTATE_REDUCE_BEFORE(&team->t.t_bar);
           }
         }
       }
     } else { // Blocktime is not infinite
       for (kmp_uint32 d = 0; d < thr_bar->my_level;
            ++d) { // Gather lowest level threads first
         kmp_uint32 last = tid + thr_bar->skip_per_level[d + 1],
                    skip = thr_bar->skip_per_level[d];
         if (last > nproc)
           last = nproc;
         for (child_tid = tid + skip; child_tid < (int)last; child_tid += skip) {
           kmp_info_t *child_thr = other_threads[child_tid];
           kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb;
           KA_TRACE(20, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) wait "
                         "T#%d(%d:%d) "
                         "arrived(%p) == %llu\n",
                         gtid, team->t.t_id, tid,
                         __kmp_gtid_from_tid(child_tid, team), team->t.t_id,
                         child_tid, &child_bar->b_arrived, new_state));
           kmp_flag_64<> flag(&child_bar->b_arrived, new_state);
           flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
-          ANNOTATE_BARRIER_END(child_thr);
           if (reduce) {
             KA_TRACE(100, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) += "
                            "T#%d(%d:%d)\n",
                            gtid, team->t.t_id, tid,
                            __kmp_gtid_from_tid(child_tid, team), team->t.t_id,
                            child_tid));
-            ANNOTATE_REDUCE_AFTER(reduce);
             (*reduce)(this_thr->th.th_local.reduce_data,
                       child_thr->th.th_local.reduce_data);
-            ANNOTATE_REDUCE_BEFORE(reduce);
-            ANNOTATE_REDUCE_BEFORE(&team->t.t_bar);
           }
         }
       }
     }
   }
   // All subordinates are gathered; now release parent if not primary thread
 
   if (!KMP_MASTER_TID(tid)) { // worker threads release parent in hierarchy
     KA_TRACE(20, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) releasing"
                   " T#%d(%d:%d) arrived(%p): %llu => %llu\n",
                   gtid, team->t.t_id, tid,
                   __kmp_gtid_from_tid(thr_bar->parent_tid, team), team->t.t_id,
                   thr_bar->parent_tid, &thr_bar->b_arrived, thr_bar->b_arrived,
                   thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
     /* Mark arrival to parent: After performing this write, a worker thread may
        not assume that the team is valid any more - it could be deallocated by
        the primary thread at any time. */
     if (thr_bar->my_level || __kmp_dflt_blocktime != KMP_MAX_BLOCKTIME ||
         !thr_bar->use_oncore_barrier) { // Parent is waiting on my b_arrived
       // flag; release it
-      ANNOTATE_BARRIER_BEGIN(this_thr);
       kmp_flag_64<> flag(&thr_bar->b_arrived,
                          other_threads[thr_bar->parent_tid]);
       flag.release();
     } else {
       // Leaf does special release on "offset" bits of parent's b_arrived flag
       thr_bar->b_arrived = team->t.t_bar[bt].b_arrived + KMP_BARRIER_STATE_BUMP;
       kmp_flag_oncore flag(&thr_bar->parent_bar->b_arrived,
                            thr_bar->offset + 1);
if (KMP_MASTER_TID(tid)) {
     // Wait for parent thread to release me
     if (!thr_bar->use_oncore_barrier ||
         __kmp_dflt_blocktime != KMP_MAX_BLOCKTIME || thr_bar->my_level != 0 ||
         thr_bar->team == NULL) {
       // Use traditional method of waiting on my own b_go flag
       thr_bar->wait_flag = KMP_BARRIER_OWN_FLAG;
       kmp_flag_64<> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
       flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj));
-      ANNOTATE_BARRIER_END(this_thr);
       TCW_8(thr_bar->b_go,
             KMP_INIT_BARRIER_STATE); // Reset my b_go flag for next time
     } else { // Thread barrier data is initialized, this is a leaf, blocktime is
       // infinite, not nested
       // Wait on my "offset" bits on parent's b_go flag
       thr_bar->wait_flag = KMP_BARRIER_PARENT_FLAG;
       kmp_flag_oncore flag(&thr_bar->parent_bar->b_go, KMP_BARRIER_STATE_BUMP,
                            thr_bar->offset + 1, bt,
if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
             KA_TRACE(
                 20,
                 ("__kmp_hierarchical_barrier_release: T#%d(%d:%d) releasing"
                  " T#%d(%d:%d) go(%p): %u => %u\n",
                  gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
                  team->t.t_id, child_tid, &child_bar->b_go, child_bar->b_go,
                  child_bar->b_go + KMP_BARRIER_STATE_BUMP));
             // Release child using child's b_go flag
-            ANNOTATE_BARRIER_BEGIN(child_thr);
             kmp_flag_64<> flag(&child_bar->b_go, child_thr);
             flag.release();
           }
         } else { // Release all children at once with leaf_state bits on my own
           // b_go flag
           thr_bar->b_go |= thr_bar->leaf_state;
         }
       }
if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
           kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb;
           KA_TRACE(20, ("__kmp_hierarchical_barrier_release: T#%d(%d:%d) "
                         "releasing T#%d(%d:%d) go(%p): %u => %u\n",
                         gtid, team->t.t_id, tid,
                         __kmp_gtid_from_tid(child_tid, team), team->t.t_id,
                         child_tid, &child_bar->b_go, child_bar->b_go,
                         child_bar->b_go + KMP_BARRIER_STATE_BUMP));
           // Release child using child's b_go flag
-          ANNOTATE_BARRIER_BEGIN(child_thr);
           kmp_flag_64<> flag(&child_bar->b_go, child_thr);
           flag.release();
         }
       }
     }
 #if KMP_BARRIER_ICV_PUSH
     if (propagate_icvs && !KMP_MASTER_TID(tid))
       // non-leaves copy ICVs from fixed ICVs to local dest
#if OMPT_SUPPORT && OMPT_OPTIONAL
   ompt_data_t *my_parallel_data;
   void *return_address;
   ompt_sync_region_t barrier_kind;
 #endif
 
   KA_TRACE(15, ("__kmp_barrier: T#%d(%d:%d) has arrived\n", gtid,
                 __kmp_team_from_gtid(gtid)->t.t_id, __kmp_tid_from_gtid(gtid)));
 
-  ANNOTATE_BARRIER_BEGIN(&team->t.t_bar);
 #if OMPT_SUPPORT
   if (ompt_enabled.enabled) {
 #if OMPT_OPTIONAL
     my_task_data = OMPT_CUR_TASK_DATA(this_thr);
     my_parallel_data = OMPT_CUR_TEAM_DATA(this_thr);
     return_address = OMPT_LOAD_RETURN_ADDRESS(gtid);
     barrier_kind = __ompt_get_barrier_kind(bt, this_thr);
     if (ompt_enabled.ompt_callback_sync_region) {
if (ompt_enabled.ompt_callback_sync_region) {
       ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
           barrier_kind, ompt_scope_end, my_parallel_data, my_task_data,
           return_address);
     }
 #endif
     this_thr->th.ompt_thread_info.state = ompt_state_work_parallel;
   }
 #endif
-  ANNOTATE_BARRIER_END(&team->t.t_bar);
 
   if (cancellable)
     return (int)cancelled;
   return status;
 }
 
 // Returns 0 if primary thread, 1 if worker thread.
 int __kmp_barrier(enum barrier_type bt, int gtid, int is_split,

 void __kmp_end_split_barrier(enum barrier_type bt, int gtid) {
   KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_end_split_barrier);
   KMP_SET_THREAD_STATE_BLOCK(PLAIN_BARRIER);
   KMP_DEBUG_ASSERT(bt < bs_last_barrier);
   int tid = __kmp_tid_from_gtid(gtid);
   kmp_info_t *this_thr = __kmp_threads[gtid];
   kmp_team_t *team = this_thr->th.th_team;
 
-  ANNOTATE_BARRIER_BEGIN(&team->t.t_bar);
   if (!team->t.t_serialized) {
     if (KMP_MASTER_GTID(gtid)) {
       switch (__kmp_barrier_release_pattern[bt]) {
       case bp_hyper_bar: {
         KMP_ASSERT(__kmp_barrier_release_branch_bits[bt]);
         __kmp_hyper_barrier_release(bt, this_thr, gtid, tid,
                                     FALSE USE_ITT_BUILD_ARG(NULL));
         break;
if (KMP_MASTER_GTID(gtid)) {
                                      FALSE USE_ITT_BUILD_ARG(NULL));
       }
       }
       if (__kmp_tasking_mode != tskm_immediate_exec) {
         __kmp_task_team_sync(this_thr, team);
       } // if
     }
   }
-  ANNOTATE_BARRIER_END(&team->t.t_bar);
 }
 
 void __kmp_join_barrier(int gtid) {
   KMP_TIME_PARTITIONED_BLOCK(OMP_join_barrier);
   KMP_SET_THREAD_STATE_BLOCK(FORK_JOIN_BARRIER);
 
   KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]);
 
#endif /* KMP_DEBUG */
 
   // Verify state
   KMP_DEBUG_ASSERT(TCR_PTR(this_thr->th.th_team));
   KMP_DEBUG_ASSERT(TCR_PTR(this_thr->th.th_root));
   KMP_DEBUG_ASSERT(this_thr == team->t.t_threads[tid]);
   KA_TRACE(10, ("__kmp_join_barrier: T#%d(%d:%d) arrived at join barrier\n",
                 gtid, team_id, tid));
 
-  ANNOTATE_BARRIER_BEGIN(&team->t.t_bar);
 #if OMPT_SUPPORT
   if (ompt_enabled.enabled) {
 #if OMPT_OPTIONAL
     ompt_data_t *my_task_data;
     ompt_data_t *my_parallel_data;
     void *codeptr = NULL;
     int ds_tid = this_thr->th.th_info.ds.ds_tid;
     if (KMP_MASTER_TID(ds_tid) &&
#if KMP_DEBUG
   }
 #endif /* KMP_DEBUG */
 
   // TODO now, mark worker threads as done so they may be disbanded
   KMP_MB(); // Flush all pending memory write invalidates.
   KA_TRACE(10,
            ("__kmp_join_barrier: T#%d(%d:%d) leaving\n", gtid, team_id, tid));
 
-  ANNOTATE_BARRIER_END(&team->t.t_bar);
 }
 
 // TODO release worker threads' fork barriers as we are ready instead of all at
 // once
 void __kmp_fork_barrier(int gtid, int tid) {
   KMP_TIME_PARTITIONED_BLOCK(OMP_fork_barrier);
   KMP_SET_THREAD_STATE_BLOCK(FORK_JOIN_BARRIER);
   kmp_info_t *this_thr = __kmp_threads[gtid];
   kmp_team_t *team = (tid == 0) ? this_thr->th.th_team : NULL;
 #if USE_ITT_BUILD
   void *itt_sync_obj = NULL;
 #endif /* USE_ITT_BUILD */
   if (team)
-    ANNOTATE_BARRIER_END(&team->t.t_bar);
 
   KA_TRACE(10, ("__kmp_fork_barrier: T#%d(%d:%d) has arrived\n", gtid,
                 (team != NULL) ? team->t.t_id : -1, tid));
 
   // th_team pointer only valid for primary thread here
   if (KMP_MASTER_TID(tid)) {
 #if USE_ITT_BUILD && USE_ITT_NOTIFY
     if (__itt_sync_create_ptr || KMP_ITT_DEBUG) {
#if USE_ITT_BUILD && USE_ITT_NOTIFY
   if (__itt_sync_create_ptr || KMP_ITT_DEBUG) {
     if (!KMP_MASTER_TID(tid)) {
       // Get correct barrier object
       itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier);
       __kmp_itt_barrier_finished(gtid, itt_sync_obj); // Workers call acquired
     } // (prepare called inside barrier_release)
   }
 #endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */
-  ANNOTATE_BARRIER_END(&team->t.t_bar);
   KA_TRACE(10, ("__kmp_fork_barrier: T#%d(%d:%d) is leaving\n", gtid,
                 team->t.t_id, tid));
 }
 
 void __kmp_setup_icv_copy(kmp_team_t *team, int new_nproc,
                           kmp_internal_control_t *new_icvs, ident_t *loc) {
   KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_setup_icv_copy);
 

openmp/runtime/src/kmp_config.h.cmake

 #cmakedefine01 LIBOMP_USE_HWLOC
 #define KMP_USE_HWLOC LIBOMP_USE_HWLOC
 #cmakedefine01 LIBOMP_ENABLE_SHARED
 #define KMP_DYNAMIC_LIB LIBOMP_ENABLE_SHARED
 #define KMP_ARCH_STR "@LIBOMP_LEGAL_ARCH@"
 #define KMP_LIBRARY_FILE "@LIBOMP_LIB_FILE@"
 #define KMP_VERSION_MAJOR @LIBOMP_VERSION_MAJOR@
 #define KMP_VERSION_MINOR @LIBOMP_VERSION_MINOR@
-#cmakedefine01 LIBOMP_TSAN_SUPPORT
-#if LIBOMP_TSAN_SUPPORT
-#define TSAN_SUPPORT
-#endif
 #cmakedefine01 MSVC
 #define KMP_MSVC_COMPAT MSVC
 #cmakedefine01 LIBOMP_HAVE_WAITPKG_INTRINSICS
 #define KMP_HAVE_WAITPKG_INTRINSICS LIBOMP_HAVE_WAITPKG_INTRINSICS
 #cmakedefine01 LIBOMP_HAVE_RTM_INTRINSICS
 #define KMP_HAVE_RTM_INTRINSICS LIBOMP_HAVE_RTM_INTRINSICS
 #cmakedefine01 LIBOMP_HAVE_IMMINTRIN_H
 #define KMP_HAVE_IMMINTRIN_H LIBOMP_HAVE_IMMINTRIN_H

openmp/runtime/src/kmp_lock.cpp

 #include "kmp.h"
 #include "kmp_i18n.h"
 #include "kmp_io.h"
 #include "kmp_itt.h"
 #include "kmp_lock.h"
 #include "kmp_wait_release.h"
 #include "kmp_wrapper_getpid.h"
 
-#include "tsan_annotations.h"
-
 #if KMP_USE_FUTEX
 #include <sys/syscall.h>
 #include <unistd.h>
 // We should really include <futex.h>, but that causes compatibility problems on
 // different Linux* OS distributions that either require that you include (or
 // break when you try to include) <pci/types.h>. Since all we need is the two
 // macros below (which are part of the kernel ABI, so can't change) we just
 // define the constants here and don't include <futex.h>
#endif /* USE_LOCK_PROFILE */
   } while (KMP_ATOMIC_LD_RLX(&lck->lk.poll) != tas_free ||
            !__kmp_atomic_compare_store_acq(&lck->lk.poll, tas_free, tas_busy));
   KMP_FSYNC_ACQUIRED(lck);
   return KMP_LOCK_ACQUIRED_FIRST;
 }
 
 int __kmp_acquire_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
   int retval = __kmp_acquire_tas_lock_timed_template(lck, gtid);
-  ANNOTATE_TAS_ACQUIRED(lck);
   return retval;
 }
 
 static int __kmp_acquire_tas_lock_with_checks(kmp_tas_lock_t *lck,
                                               kmp_int32 gtid) {
   char const *const func = "omp_set_lock";
   if ((sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
       __kmp_is_tas_lock_nestable(lck)) {
static int __kmp_test_tas_lock_with_checks(kmp_tas_lock_t *lck,
   }
   return __kmp_test_tas_lock(lck, gtid);
 }
 
 int __kmp_release_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
   KMP_MB(); /* Flush all pending memory write invalidates.  */
 
   KMP_FSYNC_RELEASING(lck);
-  ANNOTATE_TAS_RELEASED(lck);
   KMP_ATOMIC_ST_REL(&lck->lk.poll, KMP_LOCK_FREE(tas));
   KMP_MB(); /* Flush all pending memory write invalidates.  */
 
   KMP_YIELD_OVERSUB();
   return KMP_LOCK_RELEASED;
 }
 
 static int __kmp_release_tas_lock_with_checks(kmp_tas_lock_t *lck,

 int __kmp_acquire_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
   KMP_DEBUG_ASSERT(gtid >= 0);
 
   if (__kmp_get_tas_lock_owner(lck) == gtid) {
     lck->lk.depth_locked += 1;
     return KMP_LOCK_ACQUIRED_NEXT;
   } else {
     __kmp_acquire_tas_lock_timed_template(lck, gtid);
-    ANNOTATE_TAS_ACQUIRED(lck);
     lck->lk.depth_locked = 1;
     return KMP_LOCK_ACQUIRED_FIRST;
   }
 }
 
 static int __kmp_acquire_nested_tas_lock_with_checks(kmp_tas_lock_t *lck,
                                                      kmp_int32 gtid) {
   char const *const func = "omp_set_nest_lock";
#endif /* USE_LOCK_PROFILE */
   KMP_FSYNC_ACQUIRED(lck);
   KA_TRACE(1000, ("__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d exiting\n", lck,
                   lck->lk.poll, gtid));
   return KMP_LOCK_ACQUIRED_FIRST;
 }
 
 int __kmp_acquire_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
   int retval = __kmp_acquire_futex_lock_timed_template(lck, gtid);
-  ANNOTATE_FUTEX_ACQUIRED(lck);
   return retval;
 }
 
 static int __kmp_acquire_futex_lock_with_checks(kmp_futex_lock_t *lck,
                                                 kmp_int32 gtid) {
   char const *const func = "omp_set_lock";
   if ((sizeof(kmp_futex_lock_t) <= OMP_LOCK_T_SIZE) &&
       __kmp_is_futex_lock_nestable(lck)) {

 
 int __kmp_release_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
   KMP_MB(); /* Flush all pending memory write invalidates.  */
 
   KA_TRACE(1000, ("__kmp_release_futex_lock: lck:%p(0x%x), T#%d entering\n",
                   lck, lck->lk.poll, gtid));
 
   KMP_FSYNC_RELEASING(lck);
-  ANNOTATE_FUTEX_RELEASED(lck);
 
   kmp_int32 poll_val = KMP_XCHG_FIXED32(&(lck->lk.poll), KMP_LOCK_FREE(futex));
 
   KA_TRACE(1000,
            ("__kmp_release_futex_lock: lck:%p, T#%d released poll_val = 0x%x\n",
             lck, gtid, poll_val));
 
   if (KMP_LOCK_STRIP(poll_val) & 1) {

 int __kmp_acquire_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
   KMP_DEBUG_ASSERT(gtid >= 0);
 
   if (__kmp_get_futex_lock_owner(lck) == gtid) {
     lck->lk.depth_locked += 1;
     return KMP_LOCK_ACQUIRED_NEXT;
   } else {
     __kmp_acquire_futex_lock_timed_template(lck, gtid);
-    ANNOTATE_FUTEX_ACQUIRED(lck);
     lck->lk.depth_locked = 1;
     return KMP_LOCK_ACQUIRED_FIRST;
   }
 }
 
 static int __kmp_acquire_nested_futex_lock_with_checks(kmp_futex_lock_t *lck,
                                                        kmp_int32 gtid) {
   char const *const func = "omp_set_nest_lock";
if (std::atomic_load_explicit(&lck->lk.now_serving,
     return KMP_LOCK_ACQUIRED_FIRST;
   }
   KMP_WAIT_PTR(&lck->lk.now_serving, my_ticket, __kmp_bakery_check, lck);
   return KMP_LOCK_ACQUIRED_FIRST;
 }
 
 int __kmp_acquire_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
   int retval = __kmp_acquire_ticket_lock_timed_template(lck, gtid);
-  ANNOTATE_TICKET_ACQUIRED(lck);
   return retval;
 }
 
 static int __kmp_acquire_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
                                                  kmp_int32 gtid) {
   char const *const func = "omp_set_lock";
 
   if (!std::atomic_load_explicit(&lck->lk.initialized,

 }
 
 int __kmp_release_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
   kmp_uint32 distance = std::atomic_load_explicit(&lck->lk.next_ticket,
                                                   std::memory_order_relaxed) -
                         std::atomic_load_explicit(&lck->lk.now_serving,
                                                   std::memory_order_relaxed);
 
-  ANNOTATE_TICKET_RELEASED(lck);
   std::atomic_fetch_add_explicit(&lck->lk.now_serving, 1U,
                                  std::memory_order_release);
 
   KMP_YIELD(distance >
             (kmp_uint32)(__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
   return KMP_LOCK_RELEASED;
 }
 
int __kmp_acquire_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
   KMP_DEBUG_ASSERT(gtid >= 0);
 
   if (__kmp_get_ticket_lock_owner(lck) == gtid) {
     std::atomic_fetch_add_explicit(&lck->lk.depth_locked, 1,
                                    std::memory_order_relaxed);
     return KMP_LOCK_ACQUIRED_NEXT;
   } else {
     __kmp_acquire_ticket_lock_timed_template(lck, gtid);
-    ANNOTATE_TICKET_ACQUIRED(lck);
     std::atomic_store_explicit(&lck->lk.depth_locked, 1,
                                std::memory_order_relaxed);
     std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
                                std::memory_order_relaxed);
     return KMP_LOCK_ACQUIRED_FIRST;
   }
 }
 
#endif
   KMP_ASSERT2(0, "should not get here");
   return KMP_LOCK_ACQUIRED_FIRST;
 }
 
 int __kmp_acquire_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
   KMP_DEBUG_ASSERT(gtid >= 0);
 
   int retval = __kmp_acquire_queuing_lock_timed_template<false>(lck, gtid);
-  ANNOTATE_QUEUING_ACQUIRED(lck);
   return retval;
 }
 
 static int __kmp_acquire_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
                                                   kmp_int32 gtid) {
   char const *const func = "omp_set_lock";
   if (lck->lk.initialized != lck) {
     KMP_FATAL(LockIsUninitialized, func);
#endif
   head = *head_id_p;
 
   if (head == 0) { /* nobody on queue, nobody holding */
     /* try (0,0)->(-1,0) */
     if (KMP_COMPARE_AND_STORE_ACQ32(head_id_p, 0, -1)) {
       KA_TRACE(1000,
                ("__kmp_test_queuing_lock: T#%d exiting: holding lock\n", gtid));
       KMP_FSYNC_ACQUIRED(lck);
-      ANNOTATE_QUEUING_ACQUIRED(lck);
       return TRUE;
     }
   }
 
   KA_TRACE(1000,
            ("__kmp_test_queuing_lock: T#%d exiting: without lock\n", gtid));
   return FALSE;
 }
if (this_thr->th.th_spin_here)
     __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
   if (this_thr->th.th_next_waiting != 0)
     __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
 #endif
   KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here);
   KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
 
   KMP_FSYNC_RELEASING(lck);
-  ANNOTATE_QUEUING_RELEASED(lck);
 
   while (1) {
     kmp_int32 dequeued;
     kmp_int32 head;
     kmp_int32 tail;
 
     head = *head_id_p;
 

 int __kmp_acquire_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
   KMP_DEBUG_ASSERT(gtid >= 0);
 
   if (__kmp_get_queuing_lock_owner(lck) == gtid) {
     lck->lk.depth_locked += 1;
     return KMP_LOCK_ACQUIRED_NEXT;
   } else {
     __kmp_acquire_queuing_lock_timed_template<false>(lck, gtid);
-    ANNOTATE_QUEUING_ACQUIRED(lck);
     KMP_MB();
     lck->lk.depth_locked = 1;
     KMP_MB();
     lck->lk.owner_id = gtid + 1;
     return KMP_LOCK_ACQUIRED_FIRST;
   }
 }
 
static void __kmp_acquire_adaptive_lock(kmp_adaptive_lock_t *lck,
 
   // Speculative acquisition failed, so acquire it non-speculatively.
   // Count the non-speculative acquire attempt
   lck->lk.adaptive.acquire_attempts++;
 
   __kmp_acquire_queuing_lock_timed_template<FALSE>(GET_QLK_PTR(lck), gtid);
   // We have acquired the base lock, so count that.
   KMP_INC_STAT(lck, nonSpeculativeAcquires);
-  ANNOTATE_QUEUING_ACQUIRED(lck);
 }
 
 static void __kmp_acquire_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck,
                                                     kmp_int32 gtid) {
   char const *const func = "omp_set_lock";
   if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
     KMP_FATAL(LockIsUninitialized, func);
   }
if (reconfigure) {
       lck->lk.cleanup_ticket = lck->lk.next_ticket;
     }
   }
   return KMP_LOCK_ACQUIRED_FIRST;
 }
 
 int __kmp_acquire_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
   int retval = __kmp_acquire_drdpa_lock_timed_template(lck, gtid);
-  ANNOTATE_DRDPA_ACQUIRED(lck);
   return retval;
 }
 
 static int __kmp_acquire_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
                                                 kmp_int32 gtid) {
   char const *const func = "omp_set_lock";
   if (lck->lk.initialized != lck) {
     KMP_FATAL(LockIsUninitialized, func);
int __kmp_release_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
   // Read the ticket value from the lock data struct, then the polls pointer and
   // the mask.  The polls pointer must be read before the mask!!! (See above)
   kmp_uint64 ticket = lck->lk.now_serving + 1; // non-atomic load
   std::atomic<kmp_uint64> *polls = lck->lk.polls; // atomic load
   kmp_uint64 mask = lck->lk.mask; // atomic load
   KA_TRACE(1000, ("__kmp_release_drdpa_lock: ticket #%lld released lock %p\n",
                   ticket - 1, lck));
   KMP_FSYNC_RELEASING(lck);
-  ANNOTATE_DRDPA_RELEASED(lck);
   polls[ticket & mask] = ticket; // atomic store
   return KMP_LOCK_RELEASED;
 }
 
 static int __kmp_release_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
                                                 kmp_int32 gtid) {
   char const *const func = "omp_unset_lock";
   KMP_MB(); /* in case another processor initialized lock */

 int __kmp_acquire_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
   KMP_DEBUG_ASSERT(gtid >= 0);
 
   if (__kmp_get_drdpa_lock_owner(lck) == gtid) {
     lck->lk.depth_locked += 1;
     return KMP_LOCK_ACQUIRED_NEXT;
   } else {
     __kmp_acquire_drdpa_lock_timed_template(lck, gtid);
-    ANNOTATE_DRDPA_ACQUIRED(lck);
     KMP_MB();
     lck->lk.depth_locked = 1;
     KMP_MB();
     lck->lk.owner_id = gtid + 1;
     return KMP_LOCK_ACQUIRED_FIRST;
   }
 }
 
kmp_user_lock_p __kmp_user_lock_allocate(void **user_lock, kmp_int32 gtid,
   kmp_lock_index_t index;
   KMP_DEBUG_ASSERT(user_lock);
 
   __kmp_acquire_lock(&__kmp_global_lock, gtid);
 
   if (__kmp_lock_pool == NULL) {
     // Lock pool is empty. Allocate new memory.
 
-    // ANNOTATION: Found no good way to express the syncronisation
-    // between allocation and usage, so ignore the allocation
-    ANNOTATE_IGNORE_WRITES_BEGIN();
     if (__kmp_num_locks_in_block <= 1) { // Tune this cutoff point.
       lck = (kmp_user_lock_p)__kmp_allocate(__kmp_user_lock_size);
     } else {
       lck = __kmp_lock_block_allocate();
     }
-    ANNOTATE_IGNORE_WRITES_END();
 
     // Insert lock in the table so that it can be freed in __kmp_cleanup,
     // and debugger has info on all allocated locks.
     index = __kmp_lock_table_insert(lck);
   } else {
     // Pick up lock from pool.
     lck = __kmp_lock_pool;
     index = __kmp_lock_pool->pool.index;

openmp/runtime/src/kmp_runtime.cpp

 
 /* these are temporary issues to be dealt with */
 #define KMP_USE_PRCTL 0
 
 #if KMP_OS_WINDOWS
 #include <process.h>
 #endif
 
-#include "tsan_annotations.h"
-
 #if KMP_OS_WINDOWS
 // windows does not need include files as it doesn't use shared memory
 #else
 #include <sys/mman.h>
 #include <sys/stat.h>
 #include <fcntl.h>
 #define SHM_SIZE 1024
 #endif
static void __kmp_reap_thread(kmp_info_t *thread, int is_root) {
   if (!is_root) {
     if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
       /* Assume the threads are at the fork barrier here */
       KA_TRACE(
           20, ("__kmp_reap_thread: releasing T#%d from fork barrier for reap\n",
                gtid));
       /* Need release fence here to prevent seg faults for tree forkjoin barrier
        * (GEH) */
-      ANNOTATE_HAPPENS_BEFORE(thread);
       kmp_flag_64<> flag(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go,
                          thread);
       __kmp_release_64(&flag);
     }
 
     // Terminate OS thread.
     __kmp_reap_worker(thread);
 

openmp/runtime/src/kmp_tasking.cpp

 #include "kmp_stats.h"
 #include "kmp_wait_release.h"
 #include "kmp_taskdeps.h"
 
 #if OMPT_SUPPORT
 #include "ompt-specific.h"
 #endif
 
-#include "tsan_annotations.h"
-
 /* forward declaration */
 static void __kmp_enable_tasking(kmp_task_team_t *task_team,
                                  kmp_info_t *this_thr);
 static void __kmp_alloc_task_deque(kmp_info_t *thread,
                                    kmp_thread_data_t *thread_data);
 static int __kmp_realloc_task_threads_data(kmp_info_t *thread,
                                            kmp_task_team_t *task_team);
 static void __kmp_bottom_half_finish_proxy(kmp_int32 gtid, kmp_task_t *ptask);
static void __kmp_free_task(kmp_int32 gtid, kmp_taskdata_t *taskdata,
   KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 0);
   KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 1);
   KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0);
   KMP_DEBUG_ASSERT(taskdata->td_allocated_child_tasks == 0 ||
                    taskdata->td_flags.task_serial == 1);
   KMP_DEBUG_ASSERT(taskdata->td_incomplete_child_tasks == 0);
 
   taskdata->td_flags.freed = 1;
-  ANNOTATE_HAPPENS_BEFORE(taskdata);
 // deallocate the taskdata and shared variable blocks associated with this task
 #if USE_FAST_MEMORY
   __kmp_fast_free(thread, taskdata);
 #else /* ! USE_FAST_MEMORY */
   __kmp_thread_free(thread, taskdata);
 #endif
   KA_TRACE(20, ("__kmp_free_task: T#%d freed task %p\n", gtid, taskdata));
 }
kmp_task_t *__kmp_task_alloc(ident_t *loc_ref, kmp_int32 gtid,
   // Avoid double allocation here by combining shareds with taskdata
 #if USE_FAST_MEMORY
   taskdata = (kmp_taskdata_t *)__kmp_fast_allocate(
       encountering_thread, shareds_offset + sizeof_shareds);
 #else /* ! USE_FAST_MEMORY */
   taskdata = (kmp_taskdata_t *)__kmp_thread_malloc(
       encountering_thread, shareds_offset + sizeof_shareds);
 #endif /* USE_FAST_MEMORY */
-  ANNOTATE_HAPPENS_AFTER(taskdata);
 
   task = KMP_TASKDATA_TO_TASK(taskdata);
 
 // Make sure task & taskdata are aligned appropriately
 #if KMP_ARCH_X86 || KMP_ARCH_PPC64 || !KMP_HAVE_QUAD
   KMP_DEBUG_ASSERT((((kmp_uintptr_t)taskdata) & (sizeof(double) - 1)) == 0);
   KMP_DEBUG_ASSERT((((kmp_uintptr_t)task) & (sizeof(double) - 1)) == 0);
 #else
#endif
   if (flags->hidden_helper) {
     taskdata->td_flags.task_serial = FALSE;
     // Increment the number of hidden helper tasks to be executed
     KMP_ATOMIC_INC(&__kmp_unexecuted_hidden_helper_tasks);
   }
 
   KA_TRACE(20, ("__kmp_task_alloc(exit): T#%d created task %p parent=%p\n",
                 gtid, taskdata, taskdata->td_parent));
-  ANNOTATE_HAPPENS_BEFORE(task);
 
   return task;
 }
 
 kmp_task_t *__kmpc_omp_task_alloc(ident_t *loc_ref, kmp_int32 gtid,
                                   kmp_int32 flags, size_t sizeof_kmp_task_t,
                                   size_t sizeof_shareds,
                                   kmp_routine_entry_t task_entry) {
#endif
   if (taskdata->td_flags.hidden_helper) {
     // Hidden helper tasks can only be executed by hidden helper threads
     KMP_ASSERT(KMP_HIDDEN_HELPER_THREAD(gtid));
     KMP_ATOMIC_DEC(&__kmp_unexecuted_hidden_helper_tasks);
   }
 
   // Proxy tasks are not handled by the runtime
   if (taskdata->td_flags.proxy != TASK_PROXY) {
-    ANNOTATE_HAPPENS_AFTER(task);
     __kmp_task_start(gtid, task, current_task); // OMPT only if not discarded
   }
 
   // TODO: cancel tasks if the parallel region has also been cancelled
   // TODO: check if this sequence can be hoisted above __kmp_task_start
   // if cancellation has been enabled for this run ...
   if (UNLIKELY(__kmp_omp_cancellation)) {
     thread = __kmp_threads[gtid];

 
 #if OMPD_SUPPORT
   if (ompd_state & OMPD_ENABLE_BP)
     ompd_bp_task_end();
 #endif
 
   // Proxy tasks are not handled by the runtime
   if (taskdata->td_flags.proxy != TASK_PROXY) {
-    ANNOTATE_HAPPENS_BEFORE(taskdata->td_parent);
 #if OMPT_SUPPORT
     if (UNLIKELY(ompt_enabled.enabled)) {
       thread->th.ompt_thread_info = oldInfo;
       if (taskdata->td_flags.tiedness == TASK_TIED) {
         taskdata->ompt_task_info.frame.exit_frame = ompt_data_none;
       }
       __kmp_task_finish<true>(gtid, task, current_task);
     } else
#endif
   }
 
   KA_TRACE(
       10,
       ("__kmpc_omp_task_parts(exit): T#%d returning TASK_CURRENT_NOT_QUEUED: "
        "loc=%p task=%p, return: TASK_CURRENT_NOT_QUEUED\n",
        gtid, loc_ref, new_taskdata));
 
-  ANNOTATE_HAPPENS_BEFORE(new_task);
 #if OMPT_SUPPORT
   if (UNLIKELY(ompt_enabled.enabled)) {
     parent->ompt_task_info.frame.enter_frame = ompt_data_none;
   }
 #endif
   return TASK_CURRENT_NOT_QUEUED;
 }
 
if (new_taskdata->td_flags.proxy == TASK_PROXY ||
       __kmp_push_task(gtid, new_task) == TASK_NOT_PUSHED) // if cannot defer
   { // Execute this task immediately
     kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task;
     if (serialize_immediate)
       new_taskdata->td_flags.task_serial = 1;
     __kmp_invoke_task(gtid, new_task, current_task);
   }
 
-  ANNOTATE_HAPPENS_BEFORE(new_task);
   return TASK_CURRENT_NOT_QUEUED;
 }
 
 // __kmpc_omp_task: Wrapper around __kmp_omp_task to schedule a
 // non-thread-switchable task from the parent thread only!
 //
 // loc_ref: location of original task pragma (ignored)
 // gtid: Global Thread ID of encountering thread
if (ompt) {
         ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
             ompt_sync_region_taskwait, ompt_scope_end, my_parallel_data,
             my_task_data, return_address);
       }
       taskdata->ompt_task_info.frame.enter_frame = ompt_data_none;
     }
 #endif // OMPT_SUPPORT && OMPT_OPTIONAL
 
-    ANNOTATE_HAPPENS_AFTER(taskdata);
   }
 
   KA_TRACE(10, ("__kmpc_omp_taskwait(exit): T#%d task %p finished waiting, "
                 "returning TASK_CURRENT_NOT_QUEUED\n",
                 gtid, taskdata));
 
   return TASK_CURRENT_NOT_QUEUED;
 }
if (taskgroup->reduce_data != NULL &&
     }
   }
   // Restore parent taskgroup for the current task
   taskdata->td_taskgroup = taskgroup->parent;
   __kmp_thread_free(thread, taskgroup);
 
   KA_TRACE(10, ("__kmpc_end_taskgroup(exit): T#%d task %p finished waiting\n",
                 gtid, taskdata));
-  ANNOTATE_HAPPENS_AFTER(taskdata);
 
 #if OMPT_SUPPORT && OMPT_OPTIONAL
   if (UNLIKELY(ompt_enabled.ompt_callback_sync_region)) {
     ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
         ompt_sync_region_taskgroup, ompt_scope_end, &(my_parallel_data),
         &(my_task_data), codeptr);
   }
 #endif
#endif // BUILD_TIED_TASK_STACK
         __kmp_free(old_data);
       } else {
         KE_TRACE(10, ("__kmp_realloc_task_threads_data: T#%d allocating "
                       "threads data for task_team %p, size = %d\n",
                       __kmp_gtid_from_thread(thread), task_team, nthreads));
         // Make the initial allocate for threads_data array, and zero entries
         // Cannot use __kmp_thread_calloc() because threads not around for
         // kmp_reap_task_team( ).
-        ANNOTATE_IGNORE_WRITES_BEGIN();
         *threads_data_p = (kmp_thread_data_t *)__kmp_allocate(
             nthreads * sizeof(kmp_thread_data_t));
-        ANNOTATE_IGNORE_WRITES_END();
 #ifdef BUILD_TIED_TASK_STACK
         // GEH: Figure out if this is the right thing to do
         for (i = 0; i < nthreads; i++) {
           kmp_thread_data_t *thread_data = &(*threads_data_p)[i];
           __kmp_init_task_stack(__kmp_gtid_from_thread(thread), thread_data);
         }
 #endif // BUILD_TIED_TASK_STACK
       }

openmp/runtime/src/tsan_annotations.h

-/*! \file */
-/*
- * tsan_annotations.h -- ThreadSanitizer annotations to support data
- * race detection in OpenMP programs.
- */
-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef TSAN_ANNOTATIONS_H
-#define TSAN_ANNOTATIONS_H
-
-#include "kmp_config.h"
-
-/* types as used in tsan/rtl/tsan_interface_ann.cc */
-typedef unsigned long uptr;
-typedef signed long sptr;
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/* Declaration of all annotation functions in tsan/rtl/tsan_interface_ann.cc */
-void AnnotateHappensBefore(const char *f, int l, uptr addr);
-void AnnotateHappensAfter(const char *f, int l, uptr addr);
-void AnnotateCondVarSignal(const char *f, int l, uptr cv);
-void AnnotateCondVarSignalAll(const char *f, int l, uptr cv);
-void AnnotateMutexIsNotPHB(const char *f, int l, uptr mu);
-void AnnotateCondVarWait(const char *f, int l, uptr cv, uptr lock);
-void AnnotateRWLockCreate(const char *f, int l, uptr m);
-void AnnotateRWLockCreateStatic(const char *f, int l, uptr m);
-void AnnotateRWLockDestroy(const char *f, int l, uptr m);
-void AnnotateRWLockAcquired(const char *f, int l, uptr m, uptr is_w);
-void AnnotateRWLockReleased(const char *f, int l, uptr m, uptr is_w);
-void AnnotateTraceMemory(const char *f, int l, uptr mem);
-void AnnotateFlushState(const char *f, int l);
-void AnnotateNewMemory(const char *f, int l, uptr mem, uptr size);
-void AnnotateNoOp(const char *f, int l, uptr mem);
-void AnnotateFlushExpectedRaces(const char *f, int l);
-void AnnotateEnableRaceDetection(const char *f, int l, int enable);
-void AnnotateMutexIsUsedAsCondVar(const char *f, int l, uptr mu);
-void AnnotatePCQGet(const char *f, int l, uptr pcq);
-void AnnotatePCQPut(const char *f, int l, uptr pcq);
-void AnnotatePCQDestroy(const char *f, int l, uptr pcq);
-void AnnotatePCQCreate(const char *f, int l, uptr pcq);
-void AnnotateExpectRace(const char *f, int l, uptr mem, char *desc);
-void AnnotateBenignRaceSized(const char *f, int l, uptr mem, uptr size,
-                             char *desc);
-void AnnotateBenignRace(const char *f, int l, uptr mem, char *desc);
-void AnnotateIgnoreReadsBegin(const char *f, int l);
-void AnnotateIgnoreReadsEnd(const char *f, int l);
-void AnnotateIgnoreWritesBegin(const char *f, int l);
-void AnnotateIgnoreWritesEnd(const char *f, int l);
-void AnnotateIgnoreSyncBegin(const char *f, int l);
-void AnnotateIgnoreSyncEnd(const char *f, int l);
-void AnnotatePublishMemoryRange(const char *f, int l, uptr addr, uptr size);
-void AnnotateUnpublishMemoryRange(const char *f, int l, uptr addr, uptr size);
-void AnnotateThreadName(const char *f, int l, char *name);
-void WTFAnnotateHappensBefore(const char *f, int l, uptr addr);
-void WTFAnnotateHappensAfter(const char *f, int l, uptr addr);
-void WTFAnnotateBenignRaceSized(const char *f, int l, uptr mem, uptr sz,
-                                char *desc);
-int RunningOnValgrind();
-double ValgrindSlowdown(void);
-const char *ThreadSanitizerQuery(const char *query);
-void AnnotateMemoryIsInitialized(const char *f, int l, uptr mem, uptr sz);
-
-#ifdef __cplusplus
-}
-#endif
-
-#ifdef TSAN_SUPPORT
-#define ANNOTATE_HAPPENS_AFTER(addr)                                           \
-  AnnotateHappensAfter(__FILE__, __LINE__, (uptr)addr)
-#define ANNOTATE_HAPPENS_BEFORE(addr)                                          \
-  AnnotateHappensBefore(__FILE__, __LINE__, (uptr)addr)
-#define ANNOTATE_IGNORE_WRITES_BEGIN()                                         \
-  AnnotateIgnoreWritesBegin(__FILE__, __LINE__)
-#define ANNOTATE_IGNORE_WRITES_END() AnnotateIgnoreWritesEnd(__FILE__, __LINE__)
-#define ANNOTATE_RWLOCK_CREATE(lck)                                            \
-  AnnotateRWLockCreate(__FILE__, __LINE__, (uptr)lck)
-#define ANNOTATE_RWLOCK_RELEASED(lck)                                          \
-  AnnotateRWLockAcquired(__FILE__, __LINE__, (uptr)lck, 1)
-#define ANNOTATE_RWLOCK_ACQUIRED(lck)                                          \
-  AnnotateRWLockReleased(__FILE__, __LINE__, (uptr)lck, 1)
-#define ANNOTATE_BARRIER_BEGIN(addr)                                           \
-  AnnotateHappensBefore(__FILE__, __LINE__, (uptr)addr)
-#define ANNOTATE_BARRIER_END(addr)                                             \
-  AnnotateHappensAfter(__FILE__, __LINE__, (uptr)addr)
-#define ANNOTATE_REDUCE_AFTER(addr)                                            \
-  AnnotateHappensAfter(__FILE__, __LINE__, (uptr)addr)
-#define ANNOTATE_REDUCE_BEFORE(addr)                                           \
-  AnnotateHappensBefore(__FILE__, __LINE__, (uptr)addr)
-#else
-#define ANNOTATE_HAPPENS_AFTER(addr)
-#define ANNOTATE_HAPPENS_BEFORE(addr)
-#define ANNOTATE_IGNORE_WRITES_BEGIN()
-#define ANNOTATE_IGNORE_WRITES_END()
-#define ANNOTATE_RWLOCK_CREATE(lck)
-#define ANNOTATE_RWLOCK_RELEASED(lck)
-#define ANNOTATE_RWLOCK_ACQUIRED(lck)
-#define ANNOTATE_BARRIER_BEGIN(addr)
-#define ANNOTATE_BARRIER_END(addr)
-#define ANNOTATE_REDUCE_AFTER(addr)
-#define ANNOTATE_REDUCE_BEFORE(addr)
-#endif
-
-#define ANNOTATE_QUEUING
-#define ANNOTATE_TICKET
-#define ANNOTATE_FUTEX
-#define ANNOTATE_TAS
-#define ANNOTATE_DRDPA
-
-#ifdef ANNOTATE_QUEUING
-#define ANNOTATE_QUEUING_CREATE(lck)
-#define ANNOTATE_QUEUING_RELEASED(lck) ANNOTATE_HAPPENS_BEFORE(lck)
-#define ANNOTATE_QUEUING_ACQUIRED(lck) ANNOTATE_HAPPENS_AFTER(lck)
-#else
-#define ANNOTATE_QUEUING_CREATE(lck)
-#define ANNOTATE_QUEUING_RELEASED(lck)
-#define ANNOTATE_QUEUING_ACQUIRED(lck)
-#endif
-
-#ifdef ANNOTATE_TICKET
-#define ANNOTATE_TICKET_CREATE(lck)
-#define ANNOTATE_TICKET_RELEASED(lck) ANNOTATE_HAPPENS_BEFORE(lck)
-#define ANNOTATE_TICKET_ACQUIRED(lck) ANNOTATE_HAPPENS_AFTER(lck)
-#else
-#define ANNOTATE_TICKET_CREATE(lck)
-#define ANNOTATE_TICKET_RELEASED(lck)
-#define ANNOTATE_TICKET_ACQUIRED(lck)
-#endif
-
-#ifdef ANNOTATE_FUTEX
-#define ANNOTATE_FUTEX_CREATE(lck)
-#define ANNOTATE_FUTEX_RELEASED(lck) ANNOTATE_HAPPENS_BEFORE(lck)
-#define ANNOTATE_FUTEX_ACQUIRED(lck) ANNOTATE_HAPPENS_AFTER(lck)
-#else
-#define ANNOTATE_FUTEX_CREATE(lck)
-#define ANNOTATE_FUTEX_RELEASED(lck)
-#define ANNOTATE_FUTEX_ACQUIRED(lck)
-#endif
-
-#ifdef ANNOTATE_TAS
-#define ANNOTATE_TAS_CREATE(lck)
-#define ANNOTATE_TAS_RELEASED(lck) ANNOTATE_HAPPENS_BEFORE(lck)
-#define ANNOTATE_TAS_ACQUIRED(lck) ANNOTATE_HAPPENS_AFTER(lck)
-#else
-#define ANNOTATE_TAS_CREATE(lck)
-#define ANNOTATE_TAS_RELEASED(lck)
-#define ANNOTATE_TAS_ACQUIRED(lck)
-#endif
-
-#ifdef ANNOTATE_DRDPA
-#define ANNOTATE_DRDPA_CREATE(lck)
-#define ANNOTATE_DRDPA_RELEASED(lck) ANNOTATE_HAPPENS_BEFORE(lck)
-#define ANNOTATE_DRDPA_ACQUIRED(lck) ANNOTATE_HAPPENS_AFTER(lck)
-#else
-#define ANNOTATE_DRDPA_CREATE(lck)
-#define ANNOTATE_DRDPA_RELEASED(lck)
-#define ANNOTATE_DRDPA_ACQUIRED(lck)
-#endif
-
-#endif

openmp/runtime/src/tsan_annotations.cpp

-/*
- * tsan_annotations.cpp -- ThreadSanitizer annotations to support data
- * race detection in OpenMP programs.
- */
-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "tsan_annotations.h"
-
-#include <stdio.h>
-
-typedef unsigned long uptr;
-typedef signed long sptr;
-
-extern "C" __attribute__((weak)) void AnnotateHappensBefore(const char *f,
-                                                            int l, uptr addr) {}
-extern "C" __attribute__((weak)) void AnnotateHappensAfter(const char *f, int l,
-                                                           uptr addr) {}
-extern "C" __attribute__((weak)) void AnnotateCondVarSignal(const char *f,
-                                                            int l, uptr cv) {}
-extern "C" __attribute__((weak)) void AnnotateCondVarSignalAll(const char *f,
-                                                               int l, uptr cv) {
-}
-extern "C" __attribute__((weak)) void AnnotateMutexIsNotPHB(const char *f,
-                                                            int l, uptr mu) {}
-extern "C" __attribute__((weak)) void AnnotateCondVarWait(const char *f, int l,
-                                                          uptr cv, uptr lock) {}
-extern "C" __attribute__((weak)) void AnnotateRWLockCreate(const char *f, int l,
-                                                           uptr m) {}
-extern "C" __attribute__((weak)) void
-AnnotateRWLockCreateStatic(const char *f, int l, uptr m) {}
-extern "C" __attribute__((weak)) void AnnotateRWLockDestroy(const char *f,
-                                                            int l, uptr m) {}
-extern "C" __attribute__((weak)) void
-AnnotateRWLockAcquired(const char *f, int l, uptr m, uptr is_w) {}
-extern "C" __attribute__((weak)) void
-AnnotateRWLockReleased(const char *f, int l, uptr m, uptr is_w) {}
-extern "C" __attribute__((weak)) void AnnotateTraceMemory(const char *f, int l,
-                                                          uptr mem) {}
-extern "C" __attribute__((weak)) void AnnotateFlushState(const char *f, int l) {
-}
-extern "C" __attribute__((weak)) void AnnotateNewMemory(const char *f, int l,
-                                                        uptr mem, uptr size) {}
-extern "C" __attribute__((weak)) void AnnotateNoOp(const char *f, int l,
-                                                   uptr mem) {}
-extern "C" __attribute__((weak)) void AnnotateFlushExpectedRaces(const char *f,
-                                                                 int l) {}
-extern "C" __attribute__((weak)) void
-AnnotateEnableRaceDetection(const char *f, int l, int enable) {}
-extern "C" __attribute__((weak)) void
-AnnotateMutexIsUsedAsCondVar(const char *f, int l, uptr mu) {}
-extern "C" __attribute__((weak)) void AnnotatePCQGet(const char *f, int l,
-                                                     uptr pcq) {}
-extern "C" __attribute__((weak)) void AnnotatePCQPut(const char *f, int l,
-                                                     uptr pcq) {}
-extern "C" __attribute__((weak)) void AnnotatePCQDestroy(const char *f, int l,
-                                                         uptr pcq) {}
-extern "C" __attribute__((weak)) void AnnotatePCQCreate(const char *f, int l,
-                                                        uptr pcq) {}
-extern "C" __attribute__((weak)) void AnnotateExpectRace(const char *f, int l,
-                                                         uptr mem, char *desc) {
-}
-extern "C" __attribute__((weak)) void
-AnnotateBenignRaceSized(const char *f, int l, uptr mem, uptr size, char *desc) {
-}
-extern "C" __attribute__((weak)) void AnnotateBenignRace(const char *f, int l,
-                                                         uptr mem, char *desc) {
-}
-extern "C" __attribute__((weak)) void AnnotateIgnoreReadsBegin(const char *f,
-                                                               int l) {}
-extern "C" __attribute__((weak)) void AnnotateIgnoreReadsEnd(const char *f,
-                                                             int l) {}
-extern "C" __attribute__((weak)) void AnnotateIgnoreWritesBegin(const char *f,
-                                                                int l) {}
-extern "C" __attribute__((weak)) void AnnotateIgnoreWritesEnd(const char *f,
-                                                              int l) {}
-extern "C" __attribute__((weak)) void AnnotateIgnoreSyncBegin(const char *f,
-                                                              int l) {}
-extern "C" __attribute__((weak)) void AnnotateIgnoreSyncEnd(const char *f,
-                                                            int l) {}
-extern "C" __attribute__((weak)) void
-AnnotatePublishMemoryRange(const char *f, int l, uptr addr, uptr size) {}
-extern "C" __attribute__((weak)) void
-AnnotateUnpublishMemoryRange(const char *f, int l, uptr addr, uptr size) {}
-extern "C" __attribute__((weak)) void AnnotateThreadName(const char *f, int l,
-                                                         char *name) {}
-extern "C" __attribute__((weak)) void
-WTFAnnotateHappensBefore(const char *f, int l, uptr addr) {}
-extern "C" __attribute__((weak)) void
-WTFAnnotateHappensAfter(const char *f, int l, uptr addr) {}
-extern "C" __attribute__((weak)) void
-WTFAnnotateBenignRaceSized(const char *f, int l, uptr mem, uptr sz,
-                           char *desc) {}
-extern "C" __attribute__((weak)) int RunningOnValgrind() { return 0; }
-extern "C" __attribute__((weak)) double ValgrindSlowdown(void) { return 0; }
-extern "C" __attribute__((weak)) const char __attribute__((weak)) *
-    ThreadSanitizerQuery(const char *query) {
-  return 0;
-}
-extern "C" __attribute__((weak)) void
-AnnotateMemoryIsInitialized(const char *f, int l, uptr mem, uptr sz) {}

openmp/runtime/src/z_Linux_util.cpp

 #include <sys/types.h>
 #include <sys/sysctl.h>
 #endif
 
 #include <ctype.h>
 #include <dirent.h>
 #include <fcntl.h>
 
-#include "tsan_annotations.h"
-
 struct kmp_sys_timer {
   struct timespec start;
 };
 
 // Convert timespec to nanoseconds.
 #define TS2NS(timespec)                                                        \
   (((timespec).tv_sec * (long int)1e9) + (timespec).tv_nsec)
 
void __kmp_suspend_initialize(void) {
   int status;
   status = pthread_mutexattr_init(&__kmp_suspend_mutex_attr);
   KMP_CHECK_SYSFAIL("pthread_mutexattr_init", status);
   status = pthread_condattr_init(&__kmp_suspend_cond_attr);
   KMP_CHECK_SYSFAIL("pthread_condattr_init", status);
 }
 
 void __kmp_suspend_initialize_thread(kmp_info_t *th) {
-  ANNOTATE_HAPPENS_AFTER(&th->th.th_suspend_init_count);
   int old_value = KMP_ATOMIC_LD_RLX(&th->th.th_suspend_init_count);
   int new_value = __kmp_fork_count + 1;
   // Return if already initialized
   if (old_value == new_value)
     return;
   // Wait, then return if being initialized
   if (old_value == -1 || !__kmp_atomic_compare_store(
                              &th->th.th_suspend_init_count, old_value, -1)) {
     while (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init_count) != new_value) {
       KMP_CPU_PAUSE();
     }
   } else {
     // Claim to be the initializer and do initializations
     int status;
     status = pthread_cond_init(&th->th.th_suspend_cv.c_cond,
                                &__kmp_suspend_cond_attr);
     KMP_CHECK_SYSFAIL("pthread_cond_init", status);
     status = pthread_mutex_init(&th->th.th_suspend_mx.m_mutex,
                                 &__kmp_suspend_mutex_attr);
     KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
     KMP_ATOMIC_ST_REL(&th->th.th_suspend_init_count, new_value);
-    ANNOTATE_HAPPENS_BEFORE(&th->th.th_suspend_init_count);
   }
 }
 
 void __kmp_suspend_uninitialize_thread(kmp_info_t *th) {
   if (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init_count) > __kmp_fork_count) {
     /* this means we have initialize the suspension pthread objects for this
        thread in this instance of the process */
     int status;