Index: runtime/CMakeLists.txt =================================================================== --- runtime/CMakeLists.txt +++ runtime/CMakeLists.txt @@ -284,8 +284,18 @@ libomp_error_say("OpenMP Tools Interface requested but not available") endif() +# TSAN-support +set(LIBOMP_TSAN_SUPPORT FALSE CACHE BOOL + "TSAN-support?") + +set(LIBOMP_TSAN_NAME) +if(${LIBOMP_TSAN_SUPPORT}) + add_definitions(-D TSAN_SUPPORT=1 -D DYNAMIC_ANNOTATIONS_ENABLED=1) + set(LIBOMP_TSAN_NAME _tsan) +endif() + # Setting final library name -set(LIBOMP_DEFAULT_LIB_NAME libomp) +set(LIBOMP_DEFAULT_LIB_NAME libomp${LIBOMP_TSAN_NAME}) if(${PROFILE_LIBRARY}) set(LIBOMP_DEFAULT_LIB_NAME ${LIBOMP_DEFAULT_LIB_NAME}prof) endif() @@ -322,6 +332,7 @@ endif() libomp_say("Use Adaptive locks -- ${LIBOMP_USE_ADAPTIVE_LOCKS}") libomp_say("Use quad precision -- ${LIBOMP_USE_QUAD_PRECISION}") + libomp_say("Use TSAN-support -- ${LIBOMP_TSAN_SUPPORT}") endif() add_subdirectory(src) Index: runtime/src/dynamic_annotations.h =================================================================== --- /dev/null +++ runtime/src/dynamic_annotations.h @@ -0,0 +1,661 @@ +/* Copyright (c) 2011, Google Inc. +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are +* met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Neither the name of Google Inc. nor the names of its +* contributors may be used to endorse or promote products derived from +* this software without specific prior written permission. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/* This file defines dynamic annotations for use with dynamic analysis +tool such as valgrind, PIN, etc. + +Dynamic annotation is a source code annotation that affects +the generated code (that is, the annotation is not a comment). +Each such annotation is attached to a particular +instruction and/or to a particular object (address) in the program. + +The annotations that should be used by users are macros in all upper-case +(e.g., ANNOTATE_NEW_MEMORY). + +Actual implementation of these macros may differ depending on the +dynamic analysis tool being used. + +See http://code.google.com/p/data-race-test/ for more information. + +This file supports the following dynamic analysis tools: +- None (DYNAMIC_ANNOTATIONS_ENABLED is not defined or zero). +Macros are defined empty. +- ThreadSanitizer, Helgrind, DRD (DYNAMIC_ANNOTATIONS_ENABLED is 1). +Macros are defined as calls to non-inlinable empty functions +that are intercepted by Valgrind. */ + +#ifndef __DYNAMIC_ANNOTATIONS_H__ +#define __DYNAMIC_ANNOTATIONS_H__ + +#ifndef DYNAMIC_ANNOTATIONS_PREFIX +# define DYNAMIC_ANNOTATIONS_PREFIX +#endif + +// #ifndef DYNAMIC_ANNOTATIONS_PROVIDE_RUNNING_ON_VALGRIND +// # define DYNAMIC_ANNOTATIONS_PROVIDE_RUNNING_ON_VALGRIND 1 +// #endif + +#ifdef DYNAMIC_ANNOTATIONS_WANT_ATTRIBUTE_WEAK +# ifdef __GNUC__ +# define DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK __attribute__((weak)) +# else +/* TODO(glider): for Windows support we may want to change this macro in order +to prepend __declspec(selectany) to the annotations' declarations. */ +# error weak annotations are not supported for your compiler +# endif +#else +# define DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK +#endif + +/* The following preprocessor magic prepends the value of +DYNAMIC_ANNOTATIONS_PREFIX to annotation function names. */ +#define DYNAMIC_ANNOTATIONS_GLUE0(A, B) A##B +#define DYNAMIC_ANNOTATIONS_GLUE(A, B) DYNAMIC_ANNOTATIONS_GLUE0(A, B) +#define DYNAMIC_ANNOTATIONS_NAME(name) \ +DYNAMIC_ANNOTATIONS_GLUE(DYNAMIC_ANNOTATIONS_PREFIX, name) + +//#undef DYNAMIC_ANNOTATIONS_ENABLED +//# define DYNAMIC_ANNOTATIONS_ENABLED 0 + +#ifndef DYNAMIC_ANNOTATIONS_ENABLED +# define DYNAMIC_ANNOTATIONS_ENABLED 0 +#endif + +#if DYNAMIC_ANNOTATIONS_ENABLED != 0 + + /* ------------------------------------------------------------- +Annotations useful when implementing condition variables such as CondVar, +using conditional critical sections (Await/LockWhen) and when constructing +user-defined synchronization mechanisms. + +The annotations ANNOTATE_HAPPENS_BEFORE() and ANNOTATE_HAPPENS_AFTER() can +be used to define happens-before arcs in user-defined synchronization +mechanisms: the race detector will infer an arc from the former to the +latter when they share the same argument pointer. + +Example 1 (reference counting): + +void Unref() { +ANNOTATE_HAPPENS_BEFORE(&refcount_); +if (AtomicDecrementByOne(&refcount_) == 0) { +ANNOTATE_HAPPENS_AFTER(&refcount_); +delete this; +} +} + +Example 2 (message queue): + +void MyQueue::Put(Type *e) { +MutexLock lock(&mu_); +ANNOTATE_HAPPENS_BEFORE(e); +PutElementIntoMyQueue(e); +} + +Type *MyQueue::Get() { +MutexLock lock(&mu_); +Type *e = GetElementFromMyQueue(); +ANNOTATE_HAPPENS_AFTER(e); +return e; +} + +Note: when possible, please use the existing reference counting and message +queue implementations instead of inventing new ones. */ + + /* Report that wait on the condition variable at address "cv" has succeeded +and the lock at address "lock" is held. */ +#define ANNOTATE_CONDVAR_LOCK_WAIT(cv, lock) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateCondVarWait)(__FILE__, __LINE__, cv, lock) + + /* Report that wait on the condition variable at "cv" has succeeded. Variant +w/o lock. */ +#define ANNOTATE_CONDVAR_WAIT(cv) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateCondVarWait)(__FILE__, __LINE__, cv, NULL) + + /* Report that we are about to signal on the condition variable at address +"cv". */ +#define ANNOTATE_CONDVAR_SIGNAL(cv) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateCondVarSignal)(__FILE__, __LINE__, cv) + + /* Report that we are about to signal_all on the condition variable at address +"cv". */ +#define ANNOTATE_CONDVAR_SIGNAL_ALL(cv) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateCondVarSignalAll)(__FILE__, __LINE__, cv) + + /* Annotations for user-defined synchronization mechanisms. */ + #define ANNOTATE_HAPPENS_BEFORE(obj) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateHappensBefore)(__FILE__, __LINE__, obj) +#define ANNOTATE_HAPPENS_AFTER(obj) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateHappensAfter)(__FILE__, __LINE__, obj) + + /* DEPRECATED. Don't use it. */ + #define ANNOTATE_PUBLISH_MEMORY_RANGE(pointer, size) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotatePublishMemoryRange)(__FILE__, __LINE__, \ +pointer, size) + + /* DEPRECATED. Don't use it. */ + #define ANNOTATE_UNPUBLISH_MEMORY_RANGE(pointer, size) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateUnpublishMemoryRange)(__FILE__, __LINE__, \ +pointer, size) + + /* DEPRECATED. Don't use it. */ + #define ANNOTATE_SWAP_MEMORY_RANGE(pointer, size) \ +do { \ +ANNOTATE_UNPUBLISH_MEMORY_RANGE(pointer, size); \ +ANNOTATE_PUBLISH_MEMORY_RANGE(pointer, size); \ +} while (0) + + /* Instruct the tool to create a happens-before arc between mu->Unlock() and +mu->Lock(). This annotation may slow down the race detector and hide real +races. Normally it is used only when it would be difficult to annotate each +of the mutex's critical sections individually using the annotations above. +This annotation makes sense only for hybrid race detectors. For pure +happens-before detectors this is a no-op. For more details see +http://code.google.com/p/data-race-test/wiki/PureHappensBeforeVsHybrid . */ +#define ANNOTATE_PURE_HAPPENS_BEFORE_MUTEX(mu) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateMutexIsUsedAsCondVar)(__FILE__, __LINE__, \ +mu) + + /* Opposite to ANNOTATE_PURE_HAPPENS_BEFORE_MUTEX. +Instruct the tool to NOT create h-b arcs between Unlock and Lock, even in +pure happens-before mode. For a hybrid mode this is a no-op. */ +#define ANNOTATE_NOT_HAPPENS_BEFORE_MUTEX(mu) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateMutexIsNotPHB)(__FILE__, __LINE__, mu) + + /* Deprecated. Use ANNOTATE_PURE_HAPPENS_BEFORE_MUTEX. */ + #define ANNOTATE_MUTEX_IS_USED_AS_CONDVAR(mu) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateMutexIsUsedAsCondVar)(__FILE__, __LINE__, \ +mu) + + /* ------------------------------------------------------------- +Annotations useful when defining memory allocators, or when memory that +was protected in one way starts to be protected in another. */ + + /* Report that a new memory at "address" of size "size" has been allocated. +This might be used when the memory has been retrieved from a free list and +is about to be reused, or when a the locking discipline for a variable +changes. */ +#define ANNOTATE_NEW_MEMORY(address, size) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateNewMemory)(__FILE__, __LINE__, address, \ +size) + + /* ------------------------------------------------------------- +Annotations useful when defining FIFO queues that transfer data between +threads. */ + + /* Report that the producer-consumer queue (such as ProducerConsumerQueue) at +address "pcq" has been created. The ANNOTATE_PCQ_* annotations +should be used only for FIFO queues. For non-FIFO queues use +ANNOTATE_HAPPENS_BEFORE (for put) and ANNOTATE_HAPPENS_AFTER (for get). */ +#define ANNOTATE_PCQ_CREATE(pcq) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotatePCQCreate)(__FILE__, __LINE__, pcq) + + /* Report that the queue at address "pcq" is about to be destroyed. */ + #define ANNOTATE_PCQ_DESTROY(pcq) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotatePCQDestroy)(__FILE__, __LINE__, pcq) + + /* Report that we are about to put an element into a FIFO queue at address +"pcq". */ +#define ANNOTATE_PCQ_PUT(pcq) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotatePCQPut)(__FILE__, __LINE__, pcq) + + /* Report that we've just got an element from a FIFO queue at address +"pcq". */ +#define ANNOTATE_PCQ_GET(pcq) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotatePCQGet)(__FILE__, __LINE__, pcq) + + /* ------------------------------------------------------------- +Annotations that suppress errors. It is usually better to express the +program's synchronization using the other annotations, but these can +be used when all else fails. */ + + /* Report that we may have a benign race at "pointer", with size +"sizeof(*(pointer))". "pointer" must be a non-void* pointer. Insert at the +point where "pointer" has been allocated, preferably close to the point +where the race happens. See also ANNOTATE_BENIGN_RACE_STATIC. */ +#define ANNOTATE_BENIGN_RACE(pointer, description) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateBenignRaceSized)(__FILE__, __LINE__, \ +pointer, sizeof(*(pointer)), description) + + /* Same as ANNOTATE_BENIGN_RACE(address, description), but applies to +the memory range [address, address+size). */ +#define ANNOTATE_BENIGN_RACE_SIZED(address, size, description) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateBenignRaceSized)(__FILE__, __LINE__, \ +address, size, description) + + /* Request the analysis tool to ignore all reads in the current thread +until ANNOTATE_IGNORE_READS_END is called. +Useful to ignore intentional racey reads, while still checking +other reads and all writes. +See also ANNOTATE_UNPROTECTED_READ. */ +#define ANNOTATE_IGNORE_READS_BEGIN() \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateIgnoreReadsBegin)(__FILE__, __LINE__) + + /* Stop ignoring reads. */ + #define ANNOTATE_IGNORE_READS_END() \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateIgnoreReadsEnd)(__FILE__, __LINE__) + + /* Similar to ANNOTATE_IGNORE_READS_BEGIN, but ignore writes. */ + #define ANNOTATE_IGNORE_WRITES_BEGIN() \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateIgnoreWritesBegin)(__FILE__, __LINE__) + + /* Stop ignoring writes. */ + #define ANNOTATE_IGNORE_WRITES_END() \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateIgnoreWritesEnd)(__FILE__, __LINE__) + + /* Start ignoring all memory accesses (reads and writes). */ + #define ANNOTATE_IGNORE_READS_AND_WRITES_BEGIN() \ +do {\ +ANNOTATE_IGNORE_READS_BEGIN();\ +ANNOTATE_IGNORE_WRITES_BEGIN();\ +}while(0)\ + + /* Stop ignoring all memory accesses. */ + #define ANNOTATE_IGNORE_READS_AND_WRITES_END() \ +do {\ +ANNOTATE_IGNORE_WRITES_END();\ +ANNOTATE_IGNORE_READS_END();\ +}while(0)\ + + /* Similar to ANNOTATE_IGNORE_READS_BEGIN, but ignore synchronization events: +RWLOCK* and CONDVAR*. */ +#define ANNOTATE_IGNORE_SYNC_BEGIN() \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateIgnoreSyncBegin)(__FILE__, __LINE__) + + /* Stop ignoring sync events. */ + #define ANNOTATE_IGNORE_SYNC_END() \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateIgnoreSyncEnd)(__FILE__, __LINE__) + + + /* Enable (enable!=0) or disable (enable==0) race detection for all threads. +This annotation could be useful if you want to skip expensive race analysis +during some period of program execution, e.g. during initialization. */ +#define ANNOTATE_ENABLE_RACE_DETECTION(enable) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateEnableRaceDetection)(__FILE__, __LINE__, \ +enable) + + /* ------------------------------------------------------------- +Annotations useful for debugging. */ + + /* Request to trace every access to "address". */ + #define ANNOTATE_TRACE_MEMORY(address) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateTraceMemory)(__FILE__, __LINE__, address) + + /* Report the current thread name to a race detector. */ + #define ANNOTATE_THREAD_NAME(name) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateThreadName)(__FILE__, __LINE__, name) + + /* ------------------------------------------------------------- +Annotations useful when implementing locks. They are not +normally needed by modules that merely use locks. +The "lock" argument is a pointer to the lock object. */ + + /* Report that a lock has been created at address "lock". */ + #define ANNOTATE_RWLOCK_CREATE(lock) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateRWLockCreate)(__FILE__, __LINE__, lock) + + /* Report that the lock at address "lock" is about to be destroyed. */ + #define ANNOTATE_RWLOCK_DESTROY(lock) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateRWLockDestroy)(__FILE__, __LINE__, lock) + + /* Report that the lock at address "lock" has been acquired. +is_w=1 for writer lock, is_w=0 for reader lock. */ +#define ANNOTATE_RWLOCK_ACQUIRED(lock, is_w) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateRWLockAcquired)(__FILE__, __LINE__, lock, \ +is_w) + + /* Report that the lock at address "lock" is about to be released. */ + #define ANNOTATE_RWLOCK_RELEASED(lock, is_w) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateRWLockReleased)(__FILE__, __LINE__, lock, \ +is_w) + + /* ------------------------------------------------------------- +Annotations useful when implementing barriers. They are not +normally needed by modules that merely use barriers. +The "barrier" argument is a pointer to the barrier object. */ + + /* Report that the "barrier" has been initialized with initial "count". +If 'reinitialization_allowed' is true, initialization is allowed to happen +multiple times w/o calling barrier_destroy() */ +#define ANNOTATE_BARRIER_INIT(barrier, count, reinitialization_allowed) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateBarrierInit)(__FILE__, __LINE__, barrier, \ +count, reinitialization_allowed) + + /* Report that we are about to enter barrier_wait("barrier"). */ + #define ANNOTATE_BARRIER_WAIT_BEFORE(barrier) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateBarrierWaitBefore)(__FILE__, __LINE__, \ +barrier) + + /* Report that we just exited barrier_wait("barrier"). */ + #define ANNOTATE_BARRIER_WAIT_AFTER(barrier) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateBarrierWaitAfter)(__FILE__, __LINE__, \ +barrier) + + /* Report that the "barrier" has been destroyed. */ + #define ANNOTATE_BARRIER_DESTROY(barrier) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateBarrierDestroy)(__FILE__, __LINE__, \ +barrier) + + /* ------------------------------------------------------------- +Annotations useful for testing race detectors. */ + + /* Report that we expect a race on the variable at "address". +Use only in unit tests for a race detector. */ +#define ANNOTATE_EXPECT_RACE(address, description) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateExpectRace)(__FILE__, __LINE__, address, \ +description) + +#define ANNOTATE_FLUSH_EXPECTED_RACES() \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateFlushExpectedRaces)(__FILE__, __LINE__) + + /* A no-op. Insert where you like to test the interceptors. */ + #define ANNOTATE_NO_OP(arg) \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateNoOp)(__FILE__, __LINE__, arg) + + /* Force the race detector to flush its state. The actual effect depends on +* the implementation of the detector. */ +#define ANNOTATE_FLUSH_STATE() \ +DYNAMIC_ANNOTATIONS_NAME(AnnotateFlushState)(__FILE__, __LINE__) + + +#else /* DYNAMIC_ANNOTATIONS_ENABLED == 0 */ + +#define ANNOTATE_RWLOCK_CREATE(lock) /* empty */ +#define ANNOTATE_RWLOCK_DESTROY(lock) /* empty */ +#define ANNOTATE_RWLOCK_ACQUIRED(lock, is_w) /* empty */ +#define ANNOTATE_RWLOCK_RELEASED(lock, is_w) /* empty */ +#define ANNOTATE_BARRIER_INIT(barrier, count, reinitialization_allowed) /* */ +#define ANNOTATE_BARRIER_WAIT_BEFORE(barrier) /* empty */ +#define ANNOTATE_BARRIER_WAIT_AFTER(barrier) /* empty */ +#define ANNOTATE_BARRIER_DESTROY(barrier) /* empty */ +#define ANNOTATE_CONDVAR_LOCK_WAIT(cv, lock) /* empty */ +#define ANNOTATE_CONDVAR_WAIT(cv) /* empty */ +#define ANNOTATE_CONDVAR_SIGNAL(cv) /* empty */ +#define ANNOTATE_CONDVAR_SIGNAL_ALL(cv) /* empty */ +#define ANNOTATE_HAPPENS_BEFORE(obj) /* empty */ +#define ANNOTATE_HAPPENS_AFTER(obj) /* empty */ +#define ANNOTATE_PUBLISH_MEMORY_RANGE(address, size) /* empty */ +#define ANNOTATE_UNPUBLISH_MEMORY_RANGE(address, size) /* empty */ +#define ANNOTATE_SWAP_MEMORY_RANGE(address, size) /* empty */ +#define ANNOTATE_PCQ_CREATE(pcq) /* empty */ +#define ANNOTATE_PCQ_DESTROY(pcq) /* empty */ +#define ANNOTATE_PCQ_PUT(pcq) /* empty */ +#define ANNOTATE_PCQ_GET(pcq) /* empty */ +#define ANNOTATE_NEW_MEMORY(address, size) /* empty */ +#define ANNOTATE_EXPECT_RACE(address, description) /* empty */ +#define ANNOTATE_FLUSH_EXPECTED_RACES(address, description) /* empty */ +#define ANNOTATE_BENIGN_RACE(address, description) /* empty */ +#define ANNOTATE_BENIGN_RACE_SIZED(address, size, description) /* empty */ +#define ANNOTATE_PURE_HAPPENS_BEFORE_MUTEX(mu) /* empty */ +#define ANNOTATE_MUTEX_IS_USED_AS_CONDVAR(mu) /* empty */ +#define ANNOTATE_TRACE_MEMORY(arg) /* empty */ +#define ANNOTATE_THREAD_NAME(name) /* empty */ +#define ANNOTATE_IGNORE_READS_BEGIN() /* empty */ +#define ANNOTATE_IGNORE_READS_END() /* empty */ +#define ANNOTATE_IGNORE_WRITES_BEGIN() /* empty */ +#define ANNOTATE_IGNORE_WRITES_END() /* empty */ +#define ANNOTATE_IGNORE_READS_AND_WRITES_BEGIN() /* empty */ +#define ANNOTATE_IGNORE_READS_AND_WRITES_END() /* empty */ +#define ANNOTATE_IGNORE_SYNC_BEGIN() /* empty */ +#define ANNOTATE_IGNORE_SYNC_END() /* empty */ +#define ANNOTATE_ENABLE_RACE_DETECTION(enable) /* empty */ +#define ANNOTATE_NO_OP(arg) /* empty */ +#define ANNOTATE_FLUSH_STATE() /* empty */ + +#endif /* DYNAMIC_ANNOTATIONS_ENABLED */ + +//#undef ANNOTATE_HAPPENS_BEFORE +//#undef ANNOTATE_HAPPENS_AFTER +//#define ANNOTATE_HAPPENS_BEFORE(obj) /* empty */ +//#define ANNOTATE_HAPPENS_AFTER(obj) /* empty */ + +/* specify the locks to be instrumented */ +#define ANNOTATE_QUEUING + #define ANNOTATE_TICKET +#define ANNOTATE_FUTEX + #define ANNOTATE_TAS +#define ANNOTATE_DRDPA + +#ifdef ANNOTATE_QUEUING +#define ANNOTATE_QUEUING_CREATE(lck) ANNOTATE_RWLOCK_CREATE((void *) lck) +#define ANNOTATE_QUEUING_RELEASED(lck) ANNOTATE_RWLOCK_RELEASED((void *) lck, 1) +#define ANNOTATE_QUEUING_ACQUIRED(lck) ANNOTATE_RWLOCK_ACQUIRED((void *) lck, 1) +#else +#define ANNOTATE_QUEUING_CREATE(lck) (void)0 +#define ANNOTATE_QUEUING_RELEASED(lck) (void)0 +#define ANNOTATE_QUEUING_ACQUIRED(lck) (void)0 +#endif + +#ifdef ANNOTATE_TICKET +#define ANNOTATE_TICKET_CREATE(lck) ANNOTATE_RWLOCK_CREATE((void *) lck) +#define ANNOTATE_TICKET_RELEASED(lck) ANNOTATE_RWLOCK_RELEASED((void *) lck, 1) +#define ANNOTATE_TICKET_ACQUIRED(lck) ANNOTATE_RWLOCK_ACQUIRED((void *) lck, 1) +#else +#define ANNOTATE_TICKET_CREATE(lck) (void)0 +#define ANNOTATE_TICKET_RELEASED(lck) (void)0 +#define ANNOTATE_TICKET_ACQUIRED(lck) (void)0 +#endif + +#ifdef ANNOTATE_FUTEX +#define ANNOTATE_FUTEX_CREATE(lck) ANNOTATE_RWLOCK_CREATE((void *) lck) +#define ANNOTATE_FUTEX_RELEASED(lck) ANNOTATE_RWLOCK_RELEASED((void *) lck, 1) +#define ANNOTATE_FUTEX_ACQUIRED(lck) ANNOTATE_RWLOCK_ACQUIRED((void *) lck, 1) +#else +#define ANNOTATE_FUTEX_CREATE(lck) (void)0 +#define ANNOTATE_FUTEX_RELEASED(lck) (void)0 +#define ANNOTATE_FUTEX_ACQUIRED(lck) (void)0 +#endif + +#ifdef ANNOTATE_TAS +#define ANNOTATE_TAS_CREATE(lck) ANNOTATE_RWLOCK_CREATE((void *) lck) +#define ANNOTATE_TAS_RELEASED(lck) ANNOTATE_RWLOCK_RELEASED((void *) lck, 1) +#define ANNOTATE_TAS_ACQUIRED(lck) ANNOTATE_RWLOCK_ACQUIRED((void *) lck, 1) +#else +#define ANNOTATE_TAS_CREATE(lck) (void)0 +#define ANNOTATE_TAS_RELEASED(lck) (void)0 +#define ANNOTATE_TAS_ACQUIRED(lck) (void)0 +#endif + +#ifdef ANNOTATE_DRDPA +#define ANNOTATE_DRDPA_CREATE(lck) ANNOTATE_RWLOCK_CREATE((void *) lck) +#define ANNOTATE_DRDPA_RELEASED(lck) ANNOTATE_RWLOCK_RELEASED((void *) lck, 1) +#define ANNOTATE_DRDPA_ACQUIRED(lck) ANNOTATE_RWLOCK_ACQUIRED((void *) lck, 1) +#else +#define ANNOTATE_DRDPA_CREATE(lck) (void)0 +#define ANNOTATE_DRDPA_RELEASED(lck) (void)0 +#define ANNOTATE_DRDPA_ACQUIRED(lck) (void)0 +#endif + + +/* Use the macros above rather than using these functions directly. */ +#ifdef __cplusplus +extern "C" { +#endif + + +void DYNAMIC_ANNOTATIONS_NAME(AnnotateRWLockCreate)( + const char *file, int line, + const volatile void *lock) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateRWLockDestroy)( + const char *file, int line, + const volatile void *lock) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateRWLockAcquired)( + const char *file, int line, + const volatile void *lock, long is_w) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateRWLockReleased)( + const char *file, int line, + const volatile void *lock, long is_w) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateBarrierInit)( + const char *file, int line, const volatile void *barrier, long count, + long reinitialization_allowed) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateBarrierWaitBefore)( + const char *file, int line, + const volatile void *barrier) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateBarrierWaitAfter)( + const char *file, int line, + const volatile void *barrier) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateBarrierDestroy)( + const char *file, int line, + const volatile void *barrier) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateCondVarWait)( + const char *file, int line, const volatile void *cv, + const volatile void *lock) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateCondVarSignal)( + const char *file, int line, + const volatile void *cv) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateCondVarSignalAll)( + const char *file, int line, + const volatile void *cv) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateHappensBefore)( + const char *file, int line, + const volatile void *obj) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateHappensAfter)( + const char *file, int line, + const volatile void *obj) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotatePublishMemoryRange)( + const char *file, int line, + const volatile void *address, long size) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateUnpublishMemoryRange)( + const char *file, int line, + const volatile void *address, long size) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotatePCQCreate)( + const char *file, int line, + const volatile void *pcq) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotatePCQDestroy)( + const char *file, int line, + const volatile void *pcq) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotatePCQPut)( + const char *file, int line, + const volatile void *pcq) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotatePCQGet)( + const char *file, int line, + const volatile void *pcq) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateNewMemory)( + const char *file, int line, + const volatile void *mem, long size) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateExpectRace)( + const char *file, int line, const volatile void *mem, + const char *description) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateFlushExpectedRaces)( + const char *file, int line) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateBenignRace)( + const char *file, int line, const volatile void *mem, + const char *description) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateBenignRaceSized)( + const char *file, int line, const volatile void *mem, long size, + const char *description) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateMutexIsUsedAsCondVar)( + const char *file, int line, + const volatile void *mu) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateMutexIsNotPHB)( + const char *file, int line, + const volatile void *mu) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateTraceMemory)( + const char *file, int line, + const volatile void *arg) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateThreadName)( + const char *file, int line, + const char *name) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateIgnoreReadsBegin)( + const char *file, int line) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateIgnoreReadsEnd)( + const char *file, int line) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateIgnoreWritesBegin)( + const char *file, int line) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateIgnoreWritesEnd)( + const char *file, int line) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateIgnoreSyncBegin)( + const char *file, int line) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateIgnoreSyncEnd)( + const char *file, int line) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateEnableRaceDetection)( + const char *file, int line, int enable) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateNoOp)( + const char *file, int line, + const volatile void *arg) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +void DYNAMIC_ANNOTATIONS_NAME(AnnotateFlushState)( + const char *file, int line) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; + +#if DYNAMIC_ANNOTATIONS_PROVIDE_RUNNING_ON_VALGRIND == 1 +/* Return non-zero value if running under valgrind. + +If "valgrind.h" is included into dynamic_annotations.c, +the regular valgrind mechanism will be used. +See http://valgrind.org/docs/manual/manual-core-adv.html about +RUNNING_ON_VALGRIND and other valgrind "client requests". +The file "valgrind.h" may be obtained by doing +svn co svn://svn.valgrind.org/valgrind/trunk/include + +If for some reason you can't use "valgrind.h" or want to fake valgrind, +there are two ways to make this function return non-zero: +- Use environment variable: export RUNNING_ON_VALGRIND=1 +- Make your tool intercept the function RunningOnValgrind() and +change its return value. +*/ +int RunningOnValgrind(void) DYNAMIC_ANNOTATIONS_ATTRIBUTE_WEAK; +#endif /* DYNAMIC_ANNOTATIONS_PROVIDE_RUNNING_ON_VALGRIND == 1 */ + +#ifdef __cplusplus +} +#endif + +#if DYNAMIC_ANNOTATIONS_ENABLED != 0 && defined(__cplusplus) + + /* ANNOTATE_UNPROTECTED_READ is the preferred way to annotate racey reads. + +Instead of doing +ANNOTATE_IGNORE_READS_BEGIN(); +... = x; +ANNOTATE_IGNORE_READS_END(); +one can use +... = ANNOTATE_UNPROTECTED_READ(x); */ + template + inline T ANNOTATE_UNPROTECTED_READ(const volatile T &x) { + ANNOTATE_IGNORE_READS_BEGIN(); + T res = x; + ANNOTATE_IGNORE_READS_END(); + return res; + } + /* Apply ANNOTATE_BENIGN_RACE_SIZED to a static variable. */ + #define ANNOTATE_BENIGN_RACE_STATIC(static_var, description) \ +namespace { \ +class static_var ## _annotator { \ +public: \ +static_var ## _annotator() { \ +ANNOTATE_BENIGN_RACE_SIZED(&static_var, \ +sizeof(static_var), \ +# static_var ": " description); \ +} \ +}; \ +static static_var ## _annotator the ## static_var ## _annotator;\ +} +#else /* DYNAMIC_ANNOTATIONS_ENABLED == 0 */ + +#define ANNOTATE_UNPROTECTED_READ(x) (x) +#define ANNOTATE_BENIGN_RACE_STATIC(static_var, description) /* empty */ + +#endif /* DYNAMIC_ANNOTATIONS_ENABLED */ + +#endif /* __DYNAMIC_ANNOTATIONS_H__ */ Index: runtime/src/kmp_barrier.cpp =================================================================== --- runtime/src/kmp_barrier.cpp +++ runtime/src/kmp_barrier.cpp @@ -23,6 +23,10 @@ #define USE_NGO_STORES 1 #endif // KMP_MIC +#if TSAN_SUPPORT +#include "dynamic_annotations.h" +#endif + #if KMP_MIC && USE_NGO_STORES // ICV copying #define ngo_load(src) __m512d Vt = _mm512_load_pd((void *)(src)) @@ -70,6 +74,9 @@ // Mark arrival to master thread /* After performing this write, a worker thread may not assume that the team is valid any more - it could be deallocated by the master thread at any time. */ +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_BEFORE(this_thr); +#endif kmp_flag_64 flag(&thr_bar->b_arrived, other_threads[0]); flag.release(); } else { @@ -95,6 +102,9 @@ kmp_flag_64 flag(&other_threads[i]->th.th_bar[bt].bb.b_arrived, new_state); flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj) ); +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_AFTER(other_threads[ i ]); +#endif #if USE_ITT_BUILD && USE_ITT_NOTIFY // Barrier imbalance - write min of the thread time and the other thread time to the thread. if (__kmp_forkjoin_frames_mode == 2) { @@ -168,6 +178,9 @@ &other_threads[i]->th.th_bar[bt].bb.b_go, other_threads[i]->th.th_bar[bt].bb.b_go, other_threads[i]->th.th_bar[bt].bb.b_go + KMP_BARRIER_STATE_BUMP)); +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_BEFORE(other_threads[i]); +#endif kmp_flag_64 flag(&other_threads[i]->th.th_bar[bt].bb.b_go, other_threads[i]); flag.release(); } @@ -178,6 +191,9 @@ kmp_flag_64 flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP); flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj) ); +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_AFTER(this_thr); +#endif #if USE_ITT_BUILD && USE_ITT_NOTIFY if ((__itt_sync_create_ptr && itt_sync_obj == NULL) || KMP_ITT_DEBUG) { // In a fork barrier; cannot get the object reliably (or ITTNOTIFY is disabled) @@ -261,6 +277,9 @@ kmp_flag_64 flag(&child_bar->b_arrived, new_state); flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj) ); +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_AFTER(child_thr); +#endif #if USE_ITT_BUILD && USE_ITT_NOTIFY // Barrier imbalance - write min of the thread time and a child time to the thread. if (__kmp_forkjoin_frames_mode == 2) { @@ -292,6 +311,9 @@ // Mark arrival to parent thread /* After performing this write, a worker thread may not assume that the team is valid any more - it could be deallocated by the master thread at any time. */ +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_BEFORE(this_thr); +#endif kmp_flag_64 flag(&thr_bar->b_arrived, other_threads[parent_tid]); flag.release(); } else { @@ -330,6 +352,9 @@ kmp_flag_64 flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP); flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj) ); +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_AFTER(this_thr); +#endif #if USE_ITT_BUILD && USE_ITT_NOTIFY if ((__itt_sync_create_ptr && itt_sync_obj == NULL) || KMP_ITT_DEBUG) { // In fork barrier where we could not get the object reliably (or ITTNOTIFY is disabled) @@ -398,6 +423,9 @@ child_tid, &child_bar->b_go, child_bar->b_go, child_bar->b_go + KMP_BARRIER_STATE_BUMP)); // Release child from barrier +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_BEFORE(child_thr); +#endif kmp_flag_64 flag(&child_bar->b_go, child_thr); flag.release(); child++; @@ -458,6 +486,9 @@ /* After performing this write (in the last iteration of the enclosing for loop), a worker thread may not assume that the team is valid any more - it could be deallocated by the master thread at any time. */ +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_BEFORE(this_thr); +#endif p_flag.set_waiter(other_threads[parent_tid]); p_flag.release(); break; @@ -485,6 +516,9 @@ kmp_flag_64 c_flag(&child_bar->b_arrived, new_state); c_flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj) ); +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_AFTER(child_thr); +#endif #if USE_ITT_BUILD && USE_ITT_NOTIFY // Barrier imbalance - write min of the thread time and a child time to the thread. if (__kmp_forkjoin_frames_mode == 2) { @@ -555,6 +589,9 @@ kmp_flag_64 flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP); flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj) ); +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_AFTER(this_thr); +#endif #if USE_ITT_BUILD && USE_ITT_NOTIFY if ((__itt_sync_create_ptr && itt_sync_obj == NULL) || KMP_ITT_DEBUG) { // In fork barrier where we could not get the object reliably @@ -642,6 +679,9 @@ child_tid, &child_bar->b_go, child_bar->b_go, child_bar->b_go + KMP_BARRIER_STATE_BUMP)); // Release child from barrier +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_BEFORE(child_thr); +#endif kmp_flag_64 flag(&child_bar->b_go, child_thr); flag.release(); } @@ -770,9 +810,21 @@ KA_TRACE(100, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) += T#%d(%d:%d)\n", gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team), team->t.t_id, child_tid)); +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_AFTER(other_threads[child_tid]); +#endif (*reduce)(this_thr->th.th_local.reduce_data, other_threads[child_tid]->th.th_local.reduce_data); } } +#if TSAN_SUPPORT +#ifdef DYN + else{ + for (child_tid=tid+1; child_tid<=tid+thr_bar->leaf_kids; ++child_tid) { + ANNOTATE_HAPPENS_AFTER(other_threads[child_tid]); + } + } +#endif +#endif (void) KMP_TEST_THEN_AND64((volatile kmp_int64 *)&thr_bar->b_arrived, ~(thr_bar->leaf_state)); // clear leaf_state bits } // Next, wait for higher level children on each child's b_arrived flag @@ -789,6 +841,9 @@ kmp_flag_64 flag(&child_bar->b_arrived, new_state); flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj) ); +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_AFTER(child_thr); +#endif if (reduce) { KA_TRACE(100, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) += T#%d(%d:%d)\n", gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team), @@ -812,6 +867,9 @@ kmp_flag_64 flag(&child_bar->b_arrived, new_state); flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj) ); +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_AFTER(child_thr); +#endif if (reduce) { KA_TRACE(100, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) += T#%d(%d:%d)\n", gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team), @@ -833,6 +891,9 @@ the team is valid any more - it could be deallocated by the master thread at any time. */ if (thr_bar->my_level || __kmp_dflt_blocktime != KMP_MAX_BLOCKTIME || !thr_bar->use_oncore_barrier) { // Parent is waiting on my b_arrived flag; release it +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_BEFORE(this_thr); +#endif kmp_flag_64 flag(&thr_bar->b_arrived, other_threads[thr_bar->parent_tid]); flag.release(); } @@ -878,6 +939,9 @@ kmp_flag_64 flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP); flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj) ); +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_AFTER(this_thr); +#endif TCW_8(thr_bar->b_go, KMP_INIT_BARRIER_STATE); // Reset my b_go flag for next time } else { // Thread barrier data is initialized, this is a leaf, blocktime is infinite, not nested @@ -995,6 +1059,9 @@ team->t.t_id, child_tid, &child_bar->b_go, child_bar->b_go, child_bar->b_go + KMP_BARRIER_STATE_BUMP)); // Release child using child's b_go flag +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_BEFORE(child_thr); +#endif kmp_flag_64 flag(&child_bar->b_go, child_thr); flag.release(); } @@ -1018,6 +1085,9 @@ team->t.t_id, child_tid, &child_bar->b_go, child_bar->b_go, child_bar->b_go + KMP_BARRIER_STATE_BUMP)); // Release child using child's b_go flag +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_BEFORE(child_thr); +#endif kmp_flag_64 flag(&child_bar->b_go, child_thr); flag.release(); } Index: runtime/src/kmp_lock.cpp =================================================================== --- runtime/src/kmp_lock.cpp +++ runtime/src/kmp_lock.cpp @@ -21,6 +21,10 @@ #include "kmp_lock.h" #include "kmp_io.h" +#if TSAN_SUPPORT +#include "dynamic_annotations.h" +#endif + #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) # include # include @@ -133,6 +137,9 @@ __kmp_acquire_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid ) { __kmp_acquire_tas_lock_timed_template( lck, gtid ); +#if TSAN_SUPPORT + ANNOTATE_TAS_ACQUIRED(lck); +#endif } static void @@ -177,6 +184,9 @@ KMP_MB(); /* Flush all pending memory write invalidates. */ KMP_FSYNC_RELEASING(lck); +#if TSAN_SUPPORT + ANNOTATE_TAS_RELEASED(lck); +#endif KMP_ST_REL32( &(lck->lk.poll), DYNA_LOCK_FREE(tas) ); KMP_MB(); /* Flush all pending memory write invalidates. */ @@ -251,6 +261,9 @@ } else { __kmp_acquire_tas_lock_timed_template( lck, gtid ); +#if TSAN_SUPPORT + ANNOTATE_TAS_ACQUIRED(lck); +#endif lck->lk.depth_locked = 1; } } @@ -463,6 +476,9 @@ __kmp_acquire_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid ) { __kmp_acquire_futex_lock_timed_template( lck, gtid ); +#if TSAN_SUPPORT + ANNOTATE_FUTEX_ACQUIRED(lck); +#endif } static void @@ -509,6 +525,9 @@ lck, lck->lk.poll, gtid ) ); KMP_FSYNC_RELEASING(lck); +#if TSAN_SUPPORT + ANNOTATE_FUTEX_RELEASED(lck); +#endif kmp_int32 poll_val = KMP_XCHG_FIXED32( & ( lck->lk.poll ), DYNA_LOCK_FREE(futex) ); @@ -597,6 +616,9 @@ } else { __kmp_acquire_futex_lock_timed_template( lck, gtid ); +#if TSAN_SUPPORT + ANNOTATE_FUTEX_ACQUIRED(lck); +#endif lck->lk.depth_locked = 1; } } @@ -760,6 +782,9 @@ __kmp_acquire_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid ) { __kmp_acquire_ticket_lock_timed_template( lck, gtid ); +#if TSAN_SUPPORT + ANNOTATE_TICKET_ACQUIRED(lck); +#endif } static void @@ -823,6 +848,9 @@ KMP_MB(); /* Flush all pending memory write invalidates. */ KMP_FSYNC_RELEASING(lck); +#if TSAN_SUPPORT + ANNOTATE_TICKET_RELEASED(lck); +#endif distance = ( TCR_4( lck->lk.next_ticket ) - TCR_4( lck->lk.now_serving ) ); KMP_ST_REL32( &(lck->lk.now_serving), lck->lk.now_serving + 1 ); @@ -915,6 +943,9 @@ } else { __kmp_acquire_ticket_lock_timed_template( lck, gtid ); +#if TSAN_SUPPORT + ANNOTATE_TICKET_ACQUIRED(lck); +#endif KMP_MB(); lck->lk.depth_locked = 1; KMP_MB(); @@ -1398,6 +1429,9 @@ KMP_DEBUG_ASSERT( gtid >= 0 ); __kmp_acquire_queuing_lock_timed_template( lck, gtid ); +#if TSAN_SUPPORT + ANNOTATE_QUEUING_ACQUIRED(lck); +#endif } static void @@ -1446,6 +1480,9 @@ if ( KMP_COMPARE_AND_STORE_ACQ32( head_id_p, 0, -1 ) ) { KA_TRACE( 1000, ("__kmp_test_queuing_lock: T#%d exiting: holding lock\n", gtid )); KMP_FSYNC_ACQUIRED(lck); +#if TSAN_SUPPORT + ANNOTATE_QUEUING_ACQUIRED(lck); +#endif return TRUE; } } @@ -1496,6 +1533,9 @@ KMP_DEBUG_ASSERT( this_thr->th.th_next_waiting == 0 ); KMP_FSYNC_RELEASING(lck); +#if TSAN_SUPPORT + ANNOTATE_QUEUING_RELEASED(lck); +#endif while( 1 ) { kmp_int32 dequeued; @@ -1699,6 +1739,9 @@ } else { __kmp_acquire_queuing_lock_timed_template( lck, gtid ); +#if TSAN_SUPPORT + ANNOTATE_QUEUING_ACQUIRED(lck); +#endif KMP_MB(); lck->lk.depth_locked = 1; KMP_MB(); @@ -2340,6 +2383,9 @@ __kmp_acquire_queuing_lock_timed_template( GET_QLK_PTR(lck), gtid ); // We have acquired the base lock, so count that. KMP_INC_STAT(lck,nonSpeculativeAcquires ); +#if TSAN_SUPPORT + ANNOTATE_QUEUING_ACQUIRED(lck); +#endif } static void @@ -2627,6 +2673,9 @@ __kmp_acquire_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid ) { __kmp_acquire_drdpa_lock_timed_template( lck, gtid ); +#if TSAN_SUPPORT + ANNOTATE_DRDPA_ACQUIRED(lck); +#endif } static void @@ -2719,6 +2768,9 @@ KA_TRACE(1000, ("__kmp_release_drdpa_lock: ticket #%lld released lock %p\n", ticket - 1, lck)); KMP_FSYNC_RELEASING(lck); +#if TSAN_SUPPORT + ANNOTATE_DRDPA_RELEASED(lck); +#endif KMP_ST_REL64(&(polls[ticket & mask].poll), ticket); // volatile store return KMP_LOCK_RELEASED; } @@ -2823,6 +2875,9 @@ } else { __kmp_acquire_drdpa_lock_timed_template( lck, gtid ); +#if TSAN_SUPPORT + ANNOTATE_DRDPA_ACQUIRED(lck); +#endif KMP_MB(); lck->lk.depth_locked = 1; KMP_MB(); @@ -3935,12 +3990,20 @@ if ( __kmp_lock_pool == NULL ) { // Lock pool is empty. Allocate new memory. +#if TSAN_SUPPORT + // ANNOTATION: Found no good way to express the syncronisation + // between allocation and usage, so ignore the allocation + ANNOTATE_IGNORE_WRITES_BEGIN(); +#endif if ( __kmp_num_locks_in_block <= 1 ) { // Tune this cutoff point. lck = (kmp_user_lock_p) __kmp_allocate( __kmp_user_lock_size ); } else { lck = __kmp_lock_block_allocate(); } +#if TSAN_SUPPORT + ANNOTATE_IGNORE_WRITES_END(); +#endif // Insert lock in the table so that it can be freed in __kmp_cleanup, // and debugger has info on all allocated locks. Index: runtime/src/kmp_runtime.c =================================================================== --- runtime/src/kmp_runtime.c +++ runtime/src/kmp_runtime.c @@ -38,6 +38,9 @@ #include #endif +#if TSAN_SUPPORT +#include "dynamic_annotations.h" +#endif #if defined(KMP_GOMP_COMPAT) char const __kmp_version_alt_comp[] = KMP_VERSION_PREFIX "alternative compiler support: yes"; @@ -5742,6 +5745,9 @@ /* Assume the threads are at the fork barrier here */ KA_TRACE( 20, ("__kmp_reap_thread: releasing T#%d from fork barrier for reap\n", gtid ) ); /* Need release fence here to prevent seg faults for tree forkjoin barrier (GEH) */ +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_BEFORE(thread); +#endif kmp_flag_64 flag(&thread->th.th_bar[ bs_forkjoin_barrier ].bb.b_go, thread); __kmp_release_64(&flag); }; // if Index: runtime/src/kmp_tasking.c =================================================================== --- runtime/src/kmp_tasking.c +++ runtime/src/kmp_tasking.c @@ -23,6 +23,9 @@ #include "ompt-specific.h" #endif +#if TSAN_SUPPORT +#include "dynamic_annotations.h" +#endif /* ------------------------------------------------------------------------ */ /* ------------------------------------------------------------------------ */ @@ -534,6 +537,9 @@ KMP_DEBUG_ASSERT( TCR_4(taskdata->td_incomplete_child_tasks) == 0 ); taskdata->td_flags.freed = 1; +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_BEFORE(taskdata); +#endif // deallocate the taskdata and shared variable blocks associated with this task #if USE_FAST_MEMORY __kmp_fast_free( thread, taskdata ); @@ -933,6 +939,9 @@ #else /* ! USE_FAST_MEMORY */ taskdata = (kmp_taskdata_t *) __kmp_thread_malloc( thread, shareds_offset + sizeof_shareds ); #endif /* USE_FAST_MEMORY */ +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_AFTER(taskdata); +#endif task = KMP_TASKDATA_TO_TASK(taskdata); @@ -1029,6 +1038,9 @@ KA_TRACE(20, ("__kmp_task_alloc(exit): T#%d created task %p parent=%p\n", gtid, taskdata, taskdata->td_parent) ); +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_BEFORE(task); +#endif #if OMPT_SUPPORT if (ompt_enabled) { @@ -1113,6 +1125,9 @@ // Proxy tasks are not handled by the runtime if ( taskdata->td_flags.proxy != TASK_PROXY ) #endif +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_AFTER(task); +#endif __kmp_task_start( gtid, task, current_task ); #if OMPT_SUPPORT @@ -1176,6 +1191,9 @@ // Proxy tasks are not handled by the runtime if ( taskdata->td_flags.proxy != TASK_PROXY ) #endif +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_BEFORE(taskdata->td_parent); +#endif __kmp_task_finish( gtid, task, current_task ); KA_TRACE(30, ("__kmp_invoke_task(exit): T#%d completed task %p, resuming task %p\n", @@ -1215,6 +1233,9 @@ "loc=%p task=%p, return: TASK_CURRENT_NOT_QUEUED\n", gtid, loc_ref, new_taskdata ) ); +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_BEFORE(new_task); +#endif return TASK_CURRENT_NOT_QUEUED; } @@ -1259,6 +1280,9 @@ } #endif +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_BEFORE(new_task); +#endif return TASK_CURRENT_NOT_QUEUED; } @@ -1347,6 +1371,9 @@ KA_TRACE(10, ("__kmpc_omp_taskwait(exit): T#%d task %p finished waiting, " "returning TASK_CURRENT_NOT_QUEUED\n", gtid, taskdata) ); +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_AFTER(taskdata); +#endif return TASK_CURRENT_NOT_QUEUED; } @@ -1476,6 +1503,9 @@ __kmp_thread_free( thread, taskgroup ); KA_TRACE(10, ("__kmpc_end_taskgroup(exit): T#%d task %p finished waiting\n", gtid, taskdata) ); +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_AFTER(taskdata); +#endif } #endif @@ -2247,8 +2277,14 @@ // Make the initial allocate for threads_data array, and zero entries // Cannot use __kmp_thread_calloc() because threads not around for // kmp_reap_task_team( ). +#if TSAN_SUPPORT + ANNOTATE_IGNORE_WRITES_BEGIN(); +#endif *threads_data_p = (kmp_thread_data_t *) __kmp_allocate( nthreads * sizeof(kmp_thread_data_t) ); +#if TSAN_SUPPORT + ANNOTATE_IGNORE_WRITES_END(); +#endif #ifdef BUILD_TIED_TASK_STACK // GEH: Figure out if this is the right thing to do for (i = 0; i < nthreads; i++) { Index: runtime/src/z_Linux_util.c =================================================================== --- runtime/src/z_Linux_util.c +++ runtime/src/z_Linux_util.c @@ -59,6 +59,10 @@ #include #include +#if TSAN_SUPPORT +#include "dynamic_annotations.h" +#endif + /* ------------------------------------------------------------------------ */ /* ------------------------------------------------------------------------ */ @@ -1645,6 +1649,9 @@ static void __kmp_suspend_initialize_thread( kmp_info_t *th ) { +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_AFTER(&th->th.th_suspend_init_count); +#endif if ( th->th.th_suspend_init_count <= __kmp_fork_count ) { /* this means we haven't initialized the suspension pthread objects for this thread in this instance of the process */ @@ -1654,6 +1661,9 @@ status = pthread_mutex_init( &th->th.th_suspend_mx.m_mutex, & __kmp_suspend_mutex_attr ); KMP_CHECK_SYSFAIL( "pthread_mutex_init", status ); *(volatile int*)&th->th.th_suspend_init_count = __kmp_fork_count + 1; +#if TSAN_SUPPORT + ANNOTATE_HAPPENS_BEFORE(&th->th.th_suspend_init_count); +#endif }; }