Index: CMakeLists.txt =================================================================== --- CMakeLists.txt +++ CMakeLists.txt @@ -79,5 +79,17 @@ add_subdirectory(libomptarget) endif() +set(ENABLE_OMPT_TOOLS ON) +# Currently tools are not tested well on Windows or MacOS X. +if (APPLE OR WIN32) + set(ENABLE_OMPT_TOOLS OFF) +endif() + +option(OPENMP_ENABLE_OMPT_TOOLS "Enable building ompt based tools for OpenMP." + ${ENABLE_OMPT_TOOLS}) +if (OPENMP_ENABLE_OMPT_TOOLS) + add_subdirectory(tools) +endif() + # Now that we have seen all testuites, create the check-openmp target. construct_check_openmp_target() Index: runtime/CMakeLists.txt =================================================================== --- runtime/CMakeLists.txt +++ runtime/CMakeLists.txt @@ -377,3 +377,7 @@ add_subdirectory(src) add_subdirectory(test) + +# make these variables available for tools: +set(LIBOMP_LIBRARY_DIR ${LIBOMP_LIBRARY_DIR} PARENT_SCOPE) +set(LIBOMP_INCLUDE_DIR ${LIBOMP_INCLUDE_DIR} PARENT_SCOPE) Index: runtime/src/CMakeLists.txt =================================================================== --- runtime/src/CMakeLists.txt +++ runtime/src/CMakeLists.txt @@ -156,6 +156,7 @@ else() set(LIBOMP_LIBRARY_DIR ${LIBOMP_LIBRARY_DIR} PARENT_SCOPE) endif() +set(LIBOMP_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE) # Add symbolic links to libomp if(NOT WIN32) Index: runtime/src/ompt-general.cpp =================================================================== --- runtime/src/ompt-general.cpp +++ runtime/src/ompt-general.cpp @@ -230,6 +230,20 @@ const char *sep = ":"; #endif +#if KMP_OS_UNIX + { // Non-standard: load archer tool if application is built with TSan + const char *fname = "libarcher.so"; + void *h = dlopen(fname, RTLD_LAZY); + if (h) { + start_tool = (ompt_start_tool_t)dlsym(h, "ompt_start_tool"); + if (start_tool) + ret = (*start_tool)(omp_version, runtime_version); + if (ret) + return ret; + } + } +#endif + #if KMP_OS_DARWIN // Try in the current address space ret = ompt_tool_darwin(omp_version, runtime_version); Index: tools/CMakeLists.txt =================================================================== --- /dev/null +++ tools/CMakeLists.txt @@ -0,0 +1,9 @@ +# Discover the tools that use CMake in the subdirectories. +# Note that explicit cmake invocation is required every time a new tool +# is added or removed. +file(GLOB entries *) +foreach(entry ${entries}) + if(IS_DIRECTORY ${entry} AND EXISTS ${entry}/CMakeLists.txt) + add_subdirectory(${entry}) + endif() +endforeach(entry) Index: tools/archer/CMakeLists.txt =================================================================== --- /dev/null +++ tools/archer/CMakeLists.txt @@ -0,0 +1,20 @@ +# //===----------------------------------------------------------------------===// +# // +# // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# // See https://llvm.org/LICENSE.txt for details. +# // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# // +# //===----------------------------------------------------------------------===// + + + +include_directories(${LIBOMP_INCLUDE_DIR}) + +add_library(archer SHARED ompt-tsan.cpp counter.cpp) +add_library(archer_static STATIC ompt-tsan.cpp counter.cpp) + +install(TARGETS archer archer_static + LIBRARY DESTINATION ${OPENMP_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${OPENMP_INSTALL_LIBDIR}) + +add_subdirectory(tests) Index: tools/archer/README.md =================================================================== --- /dev/null +++ tools/archer/README.md @@ -0,0 +1,221 @@ +
+

Table of Contents

+
+ +
+
+ + + + +# License + +Archer is distributed under the terms of the Apache License. + +Please see LICENSE.txt for usage terms. + +LLNL-CODE-773957 + + + +# Introduction + +**Archer** is an OMPT tool which annotates OpenMP synchronization semantics for data race +detection. +This avoids false alerts in data race detection. +Archer is automatically loaded for OpenMP applications which are compiled +with ThreadSanitizer option. + + + +# Build Archer within Clang/LLVM + +This distribution of Archer is automatically built with the OpenMP runtime +and automatically loaded by the OpenMP runtime. + + + +# Usage + + + + +## How to compile + +To use archer, compile the application with the extra flag +`-fsanitize=thread`: + + clang -O3 -g -fopenmp -fsanitize=thread app.c + clang++ -O3 -g -fopenmp -fsanitize=thread app.cpp + +To compile Fortran applications, compile with gfortran, link with clang: + + gfortran -g -c -fopenmp -fsanitize=thread app.f + clang -fopenmp -fsanitize=thread app.o -lgfortran + + + + +## Runtime Flags + +TSan runtime flags are passed via **TSAN_OPTIONS** environment variable, +we highly recommend the following option to aviod false alerts for the +OpenMP or MPI runtime implementation: + + export TSAN_OPTIONS="ignore_noninstrumented_modules=1" + + +Runtime flags are passed via **ARCHER_OPTIONS** environment variable, +different flags are separated by spaces, e.g.: + + ARCHER_OPTIONS="flush_shadow=1" ./myprogram + + + + +++ ++ ++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Flag NameDefault valueDescription
flush_shadow0Flush shadow memory at the end of an outer OpenMP parallel region. Our experiments show that this can reduce memory overhead by ~30% and runtime overhead by ~10%. This flag is useful for large OpenMP applications that typically require large amounts of memory, causing out-of-memory exceptions when checked by Archer.
print_ompt_counters0Print the number of triggered OMPT events at the end of the execution.
print_max_rss0Print the RSS memory peak at the end of the execution.
verbose0Print startup information.
enable1Use Archer runtime library during execution.
+ + + + +# Example + +Let us take the program below and follow the steps to compile and +check the program for data races. + +Suppose our program is called *myprogram.c*: + + 1 #include + 2 + 3 #define N 1000 + 4 + 5 int main (int argc, char **argv) + 6 { + 7 int a[N]; + 8 + 9 #pragma omp parallel for + 10 for (int i = 0; i < N - 1; i++) { + 11 a[i] = a[i + 1]; + 12 } + 13 } + +In case we installed Archer with the official LLVM OpenMP runtime and +ThreadSanitizer support, we compile the program as follow: + + clang-archer myprogram.c -o myprogram + +otherwise, if we installed Archer with the LLVM OpenMP runtime and +ThreadSanitizer OMPT support our compile command will look like: + + clang-archer myprogram.c -o myprogram -larcher + +Now we can run the program with the following commands: + + export OMP_NUM_THREADS=2 + ./myprogram + +Archer will output a report in case it finds data races. In our case +the report will look as follow: + + ================== + WARNING: ThreadSanitizer: data race (pid=13641) + Read of size 4 at 0x7fff79a01170 by main thread: + #0 .omp_outlined. myprogram.c:11:12 (myprogram+0x00000049b5a2) + #1 __kmp_invoke_microtask (libomp.so+0x000000077842) + #2 __libc_start_main /build/glibc-t3gR2i/glibc-2.23/csu/../csu/libc-start.c:291 (libc.so.6+0x00000002082f) + + Previous write of size 4 at 0x7fff79a01170 by thread T1: + #0 .omp_outlined. myprogram.c:11:10 (myprogram+0x00000049b5d6) + #1 __kmp_invoke_microtask (libomp.so+0x000000077842) + + Location is stack of main thread. + + Thread T1 (tid=13643, running) created by main thread at: + #0 pthread_create tsan_interceptors.cc:902:3 (myprogram+0x00000043db75) + #1 __kmp_create_worker (libomp.so+0x00000006c364) + #2 __libc_start_main /build/glibc-t3gR2i/glibc-2.23/csu/../csu/libc-start.c:291 (libc.so.6+0x00000002082f) + + SUMMARY: ThreadSanitizer: data race myprogram.c:11:12 in .omp_outlined. + ================== + ThreadSanitizer: reported 1 warnings + + + + +# Contacts and Support + +- [Google group](https://groups.google.com/forum/#!forum/archer-pruner) +- [Slack Channel](https://pruners.slack.com) + +
  • For an invitation please write an email to Simone Atzeni with a reason why you want to be part of the PRUNERS Slack Team.
+- E-Mail Contacts: + + + + Index: tools/archer/counter.h =================================================================== --- /dev/null +++ tools/archer/counter.h @@ -0,0 +1,96 @@ +/* + * counter.h -- Archer runtime library, counter header file + */ + + //===----------------------------------------------------------------------===// + // + // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + // See https://llvm.org/LICENSE.txt for details. + // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + // + //===----------------------------------------------------------------------===// + + + +#include +#include +#include + +// 240 should be enough for xeon-phi +#define MAX_THREADS 240 + +#define CACHE_LINE 128 + +#define COUNT_EVENT1(name) \ + if (this_event_counter) \ + this_event_counter->name++ +#define COUNT_EVENT2(name, scope) \ + if (this_event_counter) \ + this_event_counter->name##_##scope++ +#define COUNT_EVENT3(name, scope, kind) \ + if (this_event_counter) \ + this_event_counter->name##_##scope##_##kind++ + +typedef struct alignas(128) { + int thread_begin; // (1) thread_begin + int thread_end; // (2) thread_end + int parallel_begin; // (3) parallel_begin + int parallel_end; // (4) parallel_end + int task_create_initial; // (5) task_create: task_initial + int task_create_explicit; // task_explicit + int task_create_target; // task_target + int task_create_included; // task_included + int task_create_untied; // task_untied + int task_schedule; // (6) task_schedule + int implicit_task_scope_begin; // (7) implicit task: scope_begin + int implicit_task_scope_end; // scope_end + int mutex_released_lock; // (15) mutex_released: mutex_lock + int mutex_released_nest_lock; // mutex_nest_lock + int mutex_released_critical; // mutex_critical + int mutex_released_atomic; // mutex_atomic + int mutex_released_ordered; // mutex_ordered + int mutex_released_default; // default + int task_dependences; // (16) task_dependences + int task_dependence; // (17) task_dependence + int sync_region_scope_begin_barrier; // (21) sync_region: scope_begin: + // sync_region_barrier + int sync_region_scope_begin_reduction; // sync_region_reduction + int sync_region_scope_begin_taskwait; // sync_region_taskwait + int sync_region_scope_begin_taskgroup; // sync_region_taskgroup + int sync_region_scope_end_barrier; // scope_end: + // sync_region_barrier + int sync_region_scope_end_reduction; // sync_region_reduction + int sync_region_scope_end_taskwait; // sync_region_taskwait + int sync_region_scope_end_taskgroup; // sync_region_taskgroup + int lock_init_lock; // (22) lock_init: mutex_lock + int lock_init_nest_lock; // mutex_nest_lock + int lock_init_default; // default + int lock_destroy_lock; // (23) lock_destroy mutex_lock + int lock_destroy_nest_lock; // mutex_nest_lock + int lock_destroy_default; // default + int mutex_acquire_lock; // (24) mutex_acquire: mutex_lock + int mutex_acquire_nest_lock; // mutex_nest_lock + int mutex_acquire_critical; // mutex_critical + int mutex_acquire_atomic; // mutex_atomic + int mutex_acquire_ordered; // mutex_ordered + int mutex_acquire_default; // default + int mutex_acquired_lock; // (25) mutex_acquired: mutex_lock + int mutex_acquired_nest_lock; // mutex_nest_lock + int mutex_acquired_critical; // mutex_critical + int mutex_acquired_atomic; // mutex_atomic + int mutex_acquired_ordered; // mutex_ordered + int mutex_acquired_default; // default + int nest_lock_scope_begin; // (26) nest_lock: scope_begin + int nest_lock_scope_end; // scope_end + int flush; // (27) flush +} callback_counter_t; + +#ifdef __cplusplus +extern "C" { +#endif + +void print_callbacks(callback_counter_t *counter); + +#ifdef __cplusplus +} +#endif Index: tools/archer/counter.cpp =================================================================== --- /dev/null +++ tools/archer/counter.cpp @@ -0,0 +1,123 @@ +/* + * counter.cpp -- Archer runtime library, counting callbacks + */ + + //===----------------------------------------------------------------------===// + // + // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + // See https://llvm.org/LICENSE.txt for details. + // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + // + //===----------------------------------------------------------------------===// + + + +#include "counter.h" + +#define OUTPUT_IF_NOT_NULL(format, value) \ + if (value) \ + printf(format, value) + +void print_callbacks(callback_counter_t *counter) { + int *basecounter = (int *)counter; + int total_callbacks = 0; + for (int i = 1; i < MAX_THREADS; i++) { + // if we have at least i threads, this thread should have a thread_begin + // event: + if (counter[i].thread_begin == 0) + break; + int *threadcounter = (int *)(counter + i); + for (int j = 0; j < (int)(sizeof(callback_counter_t) / sizeof(int)); j++) + basecounter[j] += threadcounter[j]; + } + + for (int j = 0; j < (int)(sizeof(callback_counter_t) / sizeof(int)); j++) + total_callbacks += basecounter[j]; + + printf("Total callbacks: %d\n", total_callbacks); + printf("--------------------------------------\n"); + OUTPUT_IF_NOT_NULL("%5d thread_begin\n", counter[0].thread_begin); + OUTPUT_IF_NOT_NULL("%5d thread_end\n", counter[0].thread_end); + OUTPUT_IF_NOT_NULL("%5d parallel_begin\n", counter[0].parallel_begin); + OUTPUT_IF_NOT_NULL("%5d parallel_end\n", counter[0].parallel_end); + OUTPUT_IF_NOT_NULL("%5d task_create : initial\n", + counter[0].task_create_initial); + OUTPUT_IF_NOT_NULL("%5d task_create : explicit\n", + counter[0].task_create_explicit); + OUTPUT_IF_NOT_NULL("%5d task_create : target\n", + counter[0].task_create_target); + OUTPUT_IF_NOT_NULL("%5d task_create : included\n", + counter[0].task_create_included); + OUTPUT_IF_NOT_NULL("%5d task_create : untied\n", + counter[0].task_create_untied); + OUTPUT_IF_NOT_NULL("%5d task_schedule\n", counter[0].task_schedule); + OUTPUT_IF_NOT_NULL("%5d implicit_task : scope_begin\n", + counter[0].implicit_task_scope_begin); + OUTPUT_IF_NOT_NULL("%5d implicit_task : scope_end\n", + counter[0].implicit_task_scope_end); + OUTPUT_IF_NOT_NULL("%5d mutex_released_lock\n", + counter[0].mutex_released_lock); + OUTPUT_IF_NOT_NULL("%5d mutex_released_nest_lock\n", + counter[0].mutex_released_nest_lock); + OUTPUT_IF_NOT_NULL("%5d mutex_released_critical\n", + counter[0].mutex_released_critical); + OUTPUT_IF_NOT_NULL("%5d mutex_released_atomic\n", + counter[0].mutex_released_atomic); + OUTPUT_IF_NOT_NULL("%5d mutex_released_ordered\n", + counter[0].mutex_released_ordered); + OUTPUT_IF_NOT_NULL("%5d mutex_released_default\n", + counter[0].mutex_released_default); + OUTPUT_IF_NOT_NULL("%5d task_dependences\n", counter[0].task_dependences); + OUTPUT_IF_NOT_NULL("%5d task_dependence\n", counter[0].task_dependence); + OUTPUT_IF_NOT_NULL("%5d sync_region : scope_begin : barrier\n", + counter[0].sync_region_scope_begin_barrier); + OUTPUT_IF_NOT_NULL("%5d sync_region : scope_begin : taskwait\n", + counter[0].sync_region_scope_begin_taskwait); + OUTPUT_IF_NOT_NULL("%5d sync_region : scope_begin : taskgroup\n", + counter[0].sync_region_scope_begin_taskgroup); + OUTPUT_IF_NOT_NULL("%5d sync_region : scope_end : barrier\n", + counter[0].sync_region_scope_end_barrier); + OUTPUT_IF_NOT_NULL("%5d sync_region : scope_end : taskwait\n", + counter[0].sync_region_scope_end_taskwait); + OUTPUT_IF_NOT_NULL("%5d sync_region : scope_end : taskgroup\n", + counter[0].sync_region_scope_end_taskgroup); + OUTPUT_IF_NOT_NULL("%5d lock_init_lock\n", counter[0].lock_init_lock); + OUTPUT_IF_NOT_NULL("%5d lock_init_nest_lock\n", + counter[0].lock_init_nest_lock); + OUTPUT_IF_NOT_NULL("%5d lock_init_default\n", counter[0].lock_init_default); + OUTPUT_IF_NOT_NULL("%5d lock_destroy_lock\n", counter[0].lock_destroy_lock); + OUTPUT_IF_NOT_NULL("%5d lock_destroy_nest_lock\n", + counter[0].lock_destroy_nest_lock); + OUTPUT_IF_NOT_NULL("%5d lock_destroy_default\n", + counter[0].lock_destroy_default); + OUTPUT_IF_NOT_NULL("%5d mutex_acquire_lock\n", counter[0].mutex_acquire_lock); + OUTPUT_IF_NOT_NULL("%5d mutex_acquire_nest_lock\n", + counter[0].mutex_acquire_nest_lock); + OUTPUT_IF_NOT_NULL("%5d mutex_acquire_critical\n", + counter[0].mutex_acquire_critical); + OUTPUT_IF_NOT_NULL("%5d mutex_acquire_atomic\n", + counter[0].mutex_acquire_atomic); + OUTPUT_IF_NOT_NULL("%5d mutex_acquire_ordered\n", + counter[0].mutex_acquire_ordered); + OUTPUT_IF_NOT_NULL("%5d mutex_acquire_default\n", + counter[0].mutex_acquire_default); + OUTPUT_IF_NOT_NULL("%5d mutex_acquired_lock\n", + counter[0].mutex_acquired_lock); + OUTPUT_IF_NOT_NULL("%5d mutex_acquired_nest_lock\n", + counter[0].mutex_acquired_nest_lock); + OUTPUT_IF_NOT_NULL("%5d mutex_acquired_critical\n", + counter[0].mutex_acquired_critical); + OUTPUT_IF_NOT_NULL("%5d mutex_acquired_atomic\n", + counter[0].mutex_acquired_atomic); + OUTPUT_IF_NOT_NULL("%5d mutex_acquired_ordered\n", + counter[0].mutex_acquired_ordered); + OUTPUT_IF_NOT_NULL("%5d mutex_acquired_default\n", + counter[0].mutex_acquired_default); + OUTPUT_IF_NOT_NULL("%5d nest_lock_scope_begin\n", + counter[0].nest_lock_scope_begin); + OUTPUT_IF_NOT_NULL("%5d nest_lock_scope_end\n", + counter[0].nest_lock_scope_end); + OUTPUT_IF_NOT_NULL("%5d flush\n", counter[0].flush); + + return; +} Index: tools/archer/ompt-tsan.cpp =================================================================== --- /dev/null +++ tools/archer/ompt-tsan.cpp @@ -0,0 +1,989 @@ +/* + * ompt-tsan.cpp -- Archer runtime library, TSan annotations for Archer + */ + + //===----------------------------------------------------------------------===// + // + // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + // See https://llvm.org/LICENSE.txt for details. + // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + // + //===----------------------------------------------------------------------===// + + +#include "counter.h" + +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if (defined __APPLE__ && defined __MACH__) +#include +#endif + +#include +#include "omp-tools.h" + +callback_counter_t *all_counter; +__thread callback_counter_t *this_event_counter; +static int runOnTsan; +static int hasReductionCallback; + +class ArcherFlags { +public: +#if (LLVM_VERSION) >= 40 + int flush_shadow; +#endif + int print_ompt_counters; + int print_max_rss; + int verbose; + int enabled; + + ArcherFlags(const char *env) + : +#if (LLVM_VERSION) >= 40 + flush_shadow(0), +#endif + print_ompt_counters(0), print_max_rss(0), verbose(0), enabled(1) { + if (env) { + std::vector tokens; + std::string token; + std::string str(env); + std::istringstream iss(str); + while (std::getline(iss, token, ' ')) + tokens.push_back(token); + + for (std::vector::iterator it = tokens.begin(); + it != tokens.end(); ++it) { +#if (LLVM_VERSION) >= 40 + if (sscanf(it->c_str(), "flush_shadow=%d", &flush_shadow)) + continue; +#endif + if (sscanf(it->c_str(), "print_ompt_counters=%d", &print_ompt_counters)) + continue; + if (sscanf(it->c_str(), "print_max_rss=%d", &print_max_rss)) + continue; + if (sscanf(it->c_str(), "verbose=%d", &verbose)) + continue; + if (sscanf(it->c_str(), "enable=%d", &enabled)) + continue; + std::cerr << "Illegal values for ARCHER_OPTIONS variable: " << token + << std::endl; + } + } + } +}; + +#if (LLVM_VERSION) >= 40 +extern "C" { +int __attribute__((weak)) __archer_get_omp_status(); +void __attribute__((weak)) __tsan_flush_memory() {} +} +#endif +ArcherFlags *archer_flags; + +// The following definitions are pasted from "llvm/Support/Compiler.h" to allow +// the code +// to be compiled with other compilers like gcc: + +#ifndef TsanHappensBefore +// Thread Sanitizer is a tool that finds races in code. +// See http://code.google.com/p/data-race-test/wiki/DynamicAnnotations . +// tsan detects these exact functions by name. +extern "C" { +#if (defined __APPLE__ && defined __MACH__) +static void AnnotateHappensAfter(const char *file, int line, + const volatile void *cv) { + void (*fptr)(const char *, int, const volatile void *); + + fptr = (void (*)(const char *, int, const volatile void *))dlsym( + RTLD_DEFAULT, "AnnotateHappensAfter"); + (*fptr)(file, line, cv); +} +static void AnnotateHappensBefore(const char *file, int line, + const volatile void *cv) { + void (*fptr)(const char *, int, const volatile void *); + + fptr = (void (*)(const char *, int, const volatile void *))dlsym( + RTLD_DEFAULT, "AnnotateHappensBefore"); + (*fptr)(file, line, cv); +} +static void AnnotateIgnoreWritesBegin(const char *file, int line) { + void (*fptr)(const char *, int); + + fptr = (void (*)(const char *, int))dlsym(RTLD_DEFAULT, + "AnnotateIgnoreWritesBegin"); + (*fptr)(file, line); +} +static void AnnotateIgnoreWritesEnd(const char *file, int line) { + void (*fptr)(const char *, int); + + fptr = (void (*)(const char *, int))dlsym(RTLD_DEFAULT, + "AnnotateIgnoreWritesEnd"); + (*fptr)(file, line); +} +static void AnnotateNewMemory(const char *file, int line, + const volatile void *cv, size_t size) { + void (*fptr)(const char *, int, const volatile void *, size_t); + + fptr = (void (*)(const char *, int, const volatile void *, size_t))dlsym( + RTLD_DEFAULT, "AnnotateNewMemory"); + (*fptr)(file, line, cv, size); +} +static int RunningOnValgrind() { + int (*fptr)(); + + fptr = (int (*)())dlsym(RTLD_DEFAULT, "RunningOnValgrind"); + if (fptr && fptr != RunningOnValgrind) + runOnTsan = 0; + return 0; +} +#else +void __attribute__((weak)) +AnnotateHappensAfter(const char *file, int line, const volatile void *cv) {} +void __attribute__((weak)) +AnnotateHappensBefore(const char *file, int line, const volatile void *cv) {} +void __attribute__((weak)) +AnnotateIgnoreWritesBegin(const char *file, int line) {} +void __attribute__((weak)) AnnotateIgnoreWritesEnd(const char *file, int line) { +} +void __attribute__((weak)) +AnnotateNewMemory(const char *file, int line, const volatile void *cv, + size_t size) {} +int __attribute__((weak)) RunningOnValgrind() { + runOnTsan = 0; + return 0; +} +#endif +} + +// This marker is used to define a happens-before arc. The race detector will +// infer an arc from the begin to the end when they share the same pointer +// argument. +#define TsanHappensBefore(cv) AnnotateHappensBefore(__FILE__, __LINE__, cv) + +// This marker defines the destination of a happens-before arc. +#define TsanHappensAfter(cv) AnnotateHappensAfter(__FILE__, __LINE__, cv) + +// Ignore any races on writes between here and the next TsanIgnoreWritesEnd. +#define TsanIgnoreWritesBegin() AnnotateIgnoreWritesBegin(__FILE__, __LINE__) + +// Resume checking for racy writes. +#define TsanIgnoreWritesEnd() AnnotateIgnoreWritesEnd(__FILE__, __LINE__) + +// We don't really delete the clock for now +#define TsanDeleteClock(cv) + +// newMemory +#define TsanNewMemory(addr, size) \ + AnnotateNewMemory(__FILE__, __LINE__, addr, size) +#define TsanFreeMemory(addr, size) \ + AnnotateNewMemory(__FILE__, __LINE__, addr, size) +#endif + +/// Required OMPT inquiry functions. +static ompt_get_parallel_info_t ompt_get_parallel_info; +static ompt_get_thread_data_t ompt_get_thread_data; + +typedef uint64_t ompt_tsan_clockid; + +static uint64_t my_next_id() { + static uint64_t ID = 0; + uint64_t ret = __sync_fetch_and_add(&ID, 1); + return ret; +} + +// Data structure to provide a threadsafe pool of reusable objects. +// DataPool +template struct DataPool { + std::mutex DPMutex; + std::stack DataPointer; + std::list memory; + int total; + + void newDatas() { + // prefix the Data with a pointer to 'this', allows to return memory to + // 'this', + // without explicitly knowing the source. + // + // To reduce lock contention, we use thread local DataPools, but Data + // objects move to other threads. + // The strategy is to get objects from local pool. Only if the object moved + // to another + // thread, we might see a penalty on release (returnData). + // For "single producer" pattern, a single thread creates tasks, these are + // executed by other threads. + // The master will have a high demand on TaskData, so return after use. + struct pooldata { + DataPool *dp; + T data; + }; + // We alloc without initialize the memory. We cannot call constructors. + // Therfore use malloc! + pooldata *datas = (pooldata *)malloc(sizeof(pooldata) * N); + memory.push_back(datas); + for (int i = 0; i < N; i++) { + datas[i].dp = this; + DataPointer.push(&(datas[i].data)); + } + total += N; + } + + T *getData() { + T *ret; + DPMutex.lock(); + if (DataPointer.empty()) + newDatas(); + ret = DataPointer.top(); + DataPointer.pop(); + DPMutex.unlock(); + return ret; + } + + void returnData(T *data) { + DPMutex.lock(); + DataPointer.push(data); + DPMutex.unlock(); + } + + void getDatas(int n, T **datas) { + DPMutex.lock(); + for (int i = 0; i < n; i++) { + if (DataPointer.empty()) + newDatas(); + datas[i] = DataPointer.top(); + DataPointer.pop(); + } + DPMutex.unlock(); + } + + void returnDatas(int n, T **datas) { + DPMutex.lock(); + for (int i = 0; i < n; i++) { + DataPointer.push(datas[i]); + } + DPMutex.unlock(); + } + + DataPool() : DPMutex(), DataPointer(), total(0) {} + + ~DataPool() { + // we assume all memory is returned when the thread finished / destructor is + // called + for (auto i : memory) + if (i) + free(i); + } +}; + +// This function takes care to return the data to the originating DataPool +// A pointer to the originating DataPool is stored just before the actual data. +template static void retData(void *data) { + ((DataPool **)data)[-1]->returnData((T *)data); +} + +struct ParallelData; +__thread DataPool *pdp; + +/// Data structure to store additional information for parallel regions. +struct ParallelData { + + // Parallel fork is just another barrier, use Barrier[1] + + /// Two addresses for relationships with barriers. + ompt_tsan_clockid Barrier[2]; + + void *GetParallelPtr() { return &(Barrier[1]); } + + void *GetBarrierPtr(unsigned Index) { return &(Barrier[Index]); } + + ~ParallelData() { + TsanDeleteClock(&(Barrier[0])); + TsanDeleteClock(&(Barrier[1])); + } + // overload new/delete to use DataPool for memory management. + void *operator new(size_t size) { return pdp->getData(); } + void operator delete(void *p, size_t) { retData(p); } +}; + +static inline ParallelData *ToParallelData(ompt_data_t *parallel_data) { + return reinterpret_cast(parallel_data->ptr); +} + +struct Taskgroup; +__thread DataPool *tgp; + +/// Data structure to support stacking of taskgroups and allow synchronization. +struct Taskgroup { + /// Its address is used for relationships of the taskgroup's task set. + ompt_tsan_clockid Ptr; + + /// Reference to the parent taskgroup. + Taskgroup *Parent; + + Taskgroup(Taskgroup *Parent) : Parent(Parent) {} + ~Taskgroup() { TsanDeleteClock(&Ptr); } + + void *GetPtr() { return &Ptr; } + // overload new/delete to use DataPool for memory management. + void *operator new(size_t size) { return tgp->getData(); } + void operator delete(void *p, size_t) { retData(p); } +}; + +struct TaskData; +__thread DataPool *tdp; + +/// Data structure to store additional information for tasks. +struct TaskData { + /// Its address is used for relationships of this task. + ompt_tsan_clockid Task; + + /// Child tasks use its address to declare a relationship to a taskwait in + /// this task. + ompt_tsan_clockid Taskwait; + + /// Whether this task is currently executing a barrier. + bool InBarrier; + + /// Whether this task is an included task. + bool Included; + + /// Index of which barrier to use next. + char BarrierIndex; + + /// Count how often this structure has been put into child tasks + 1. + std::atomic_int RefCount; + + /// Reference to the parent that created this task. + TaskData *Parent; + + /// Reference to the implicit task in the stack above this task. + TaskData *ImplicitTask; + + /// Reference to the team of this task. + ParallelData *Team; + + /// Reference to the current taskgroup that this task either belongs to or + /// that it just created. + Taskgroup *TaskGroup; + + /// Dependency information for this task. + ompt_dependence_t *Dependencies; + + /// Number of dependency entries. + unsigned DependencyCount; + + void *PrivateData; + size_t PrivateDataSize; + + int execution; + int freed; + + TaskData(TaskData *Parent) + : InBarrier(false), Included(false), BarrierIndex(0), RefCount(1), + Parent(Parent), ImplicitTask(nullptr), Team(Parent->Team), + TaskGroup(nullptr), DependencyCount(0), execution(0), freed(0) { + if (Parent != nullptr) { + Parent->RefCount++; + // Copy over pointer to taskgroup. This task may set up its own stack + // but for now belongs to its parent's taskgroup. + TaskGroup = Parent->TaskGroup; + } + } + + TaskData(ParallelData *Team = nullptr) + : InBarrier(false), Included(false), BarrierIndex(0), RefCount(1), + Parent(nullptr), ImplicitTask(this), Team(Team), TaskGroup(nullptr), + DependencyCount(0), execution(1), freed(0) {} + + ~TaskData() { + TsanDeleteClock(&Task); + TsanDeleteClock(&Taskwait); + } + + void *GetTaskPtr() { return &Task; } + + void *GetTaskwaitPtr() { return &Taskwait; } + // overload new/delete to use DataPool for memory management. + void *operator new(size_t size) { return tdp->getData(); } + void operator delete(void *p, size_t) { retData(p); } +}; + +static inline TaskData *ToTaskData(ompt_data_t *task_data) { + return reinterpret_cast(task_data->ptr); +} + +static inline void *ToInAddr(void *OutAddr) { + // FIXME: This will give false negatives when a second variable lays directly + // behind a variable that only has a width of 1 byte. + // Another approach would be to "negate" the address or to flip the + // first bit... + return reinterpret_cast(OutAddr) + 1; +} + +/// Store a mutex for each wait_id to resolve race condition with callbacks. +std::unordered_map Locks; +std::mutex LocksMutex; + +static void ompt_tsan_thread_begin(ompt_thread_t thread_type, + ompt_data_t *thread_data) { + pdp = new DataPool; + TsanNewMemory(pdp, sizeof(pdp)); + tgp = new DataPool; + TsanNewMemory(tgp, sizeof(tgp)); + tdp = new DataPool; + TsanNewMemory(tdp, sizeof(tdp)); + thread_data->value = my_next_id(); + if (archer_flags->print_ompt_counters && thread_data->value < MAX_THREADS) + this_event_counter = &(all_counter[thread_data->value]); + else + this_event_counter = NULL; + COUNT_EVENT1(thread_begin); +} + +static void ompt_tsan_thread_end(ompt_data_t *thread_data) { + delete pdp; + delete tgp; + delete tdp; + COUNT_EVENT1(thread_end); +} + +/// OMPT event callbacks for handling parallel regions. + +static void ompt_tsan_parallel_begin(ompt_data_t *parent_task_data, + const ompt_frame_t *parent_task_frame, + ompt_data_t *parallel_data, + uint32_t requested_team_size, + int flag, + const void *codeptr_ra) { + ParallelData *Data = new ParallelData; + parallel_data->ptr = Data; + + TsanHappensBefore(Data->GetParallelPtr()); + COUNT_EVENT1(parallel_begin); +} + +static void ompt_tsan_parallel_end(ompt_data_t *parallel_data, + ompt_data_t *task_data, + int flag, + const void *codeptr_ra) { + ParallelData *Data = ToParallelData(parallel_data); + TsanHappensAfter(Data->GetBarrierPtr(0)); + TsanHappensAfter(Data->GetBarrierPtr(1)); + + delete Data; + +#if (LLVM_VERSION >= 40) + if (&__archer_get_omp_status) { + if (__archer_get_omp_status() == 0 && archer_flags->flush_shadow) + __tsan_flush_memory(); + } +#endif + + COUNT_EVENT1(parallel_end); +} + +static void ompt_tsan_implicit_task(ompt_scope_endpoint_t endpoint, + ompt_data_t *parallel_data, + ompt_data_t *task_data, + unsigned int team_size, + unsigned int thread_num, + int type) { + switch (endpoint) { + case ompt_scope_begin: + task_data->ptr = new TaskData(ToParallelData(parallel_data)); + TsanHappensAfter(ToParallelData(parallel_data)->GetParallelPtr()); + COUNT_EVENT2(implicit_task, scope_begin); + break; + case ompt_scope_end: + TaskData *Data = ToTaskData(task_data); + assert(Data->freed == 0 && "Implicit task end should only be called once!"); + Data->freed = 1; + assert(Data->RefCount == 1 && + "All tasks should have finished at the implicit barrier!"); + delete Data; + COUNT_EVENT2(implicit_task, scope_end); + break; + } +} + +static void ompt_tsan_sync_region(ompt_sync_region_t kind, + ompt_scope_endpoint_t endpoint, + ompt_data_t *parallel_data, + ompt_data_t *task_data, + const void *codeptr_ra) { + TaskData *Data = ToTaskData(task_data); + switch (endpoint) { + case ompt_scope_begin: + switch (kind) { + case ompt_sync_region_barrier_implementation: + case ompt_sync_region_barrier_implicit: + case ompt_sync_region_barrier_explicit: + case ompt_sync_region_barrier: { + char BarrierIndex = Data->BarrierIndex; + TsanHappensBefore(Data->Team->GetBarrierPtr(BarrierIndex)); + + if (hasReductionCallback < ompt_set_always) { + // We ignore writes inside the barrier. These would either occur during + // 1. reductions performed by the runtime which are guaranteed to be + // race-free. + // 2. execution of another task. + // For the latter case we will re-enable tracking in task_switch. + Data->InBarrier = true; + TsanIgnoreWritesBegin(); + } + + COUNT_EVENT3(sync_region, scope_begin, barrier); + break; + } + + case ompt_sync_region_taskwait: + COUNT_EVENT3(sync_region, scope_begin, taskwait); + break; + + case ompt_sync_region_taskgroup: + Data->TaskGroup = new Taskgroup(Data->TaskGroup); + COUNT_EVENT3(sync_region, scope_begin, taskgroup); + break; + + default: + break; + } + break; + case ompt_scope_end: + switch (kind) { + case ompt_sync_region_barrier_implementation: + case ompt_sync_region_barrier_implicit: + case ompt_sync_region_barrier_explicit: + case ompt_sync_region_barrier: { + if (hasReductionCallback < ompt_set_always) { + // We want to track writes after the barrier again. + Data->InBarrier = false; + TsanIgnoreWritesEnd(); + } + + char BarrierIndex = Data->BarrierIndex; + // Barrier will end after it has been entered by all threads. + if (parallel_data) + TsanHappensAfter(Data->Team->GetBarrierPtr(BarrierIndex)); + + // It is not guaranteed that all threads have exited this barrier before + // we enter the next one. So we will use a different address. + // We are however guaranteed that this current barrier is finished + // by the time we exit the next one. So we can then reuse the first + // address. + Data->BarrierIndex = (BarrierIndex + 1) % 2; + COUNT_EVENT3(sync_region, scope_end, barrier); + break; + } + + case ompt_sync_region_taskwait: { + COUNT_EVENT3(sync_region, scope_end, taskwait); + if (Data->execution > 1) + TsanHappensAfter(Data->GetTaskwaitPtr()); + break; + } + + case ompt_sync_region_taskgroup: { + assert(Data->TaskGroup != nullptr && + "Should have at least one taskgroup!"); + + TsanHappensAfter(Data->TaskGroup->GetPtr()); + + // Delete this allocated taskgroup, all descendent task are finished by + // now. + Taskgroup *Parent = Data->TaskGroup->Parent; + delete Data->TaskGroup; + Data->TaskGroup = Parent; + COUNT_EVENT3(sync_region, scope_end, taskgroup); + break; + } + + default: + break; + } + break; + } +} + +static void ompt_tsan_reduction(ompt_sync_region_t kind, + ompt_scope_endpoint_t endpoint, + ompt_data_t *parallel_data, + ompt_data_t *task_data, + const void *codeptr_ra) { + switch (endpoint) { + case ompt_scope_begin: + switch (kind) { + case ompt_sync_region_reduction: + TsanIgnoreWritesBegin(); + COUNT_EVENT3(sync_region, scope_begin, reduction); + break; + default: + break; + } + break; + case ompt_scope_end: + switch (kind) { + case ompt_sync_region_reduction: + TsanIgnoreWritesEnd(); + COUNT_EVENT3(sync_region, scope_begin, reduction); + break; + default: + break; + } + break; + } +} + +/// OMPT event callbacks for handling tasks. + +static void ompt_tsan_task_create( + ompt_data_t *parent_task_data, /* id of parent task */ + const ompt_frame_t *parent_frame, /* frame data for parent task */ + ompt_data_t *new_task_data, /* id of created task */ + int type, int has_dependences, + const void *codeptr_ra) /* pointer to outlined function */ +{ + TaskData *Data; + assert(new_task_data->ptr == NULL && + "Task data should be initialized to NULL"); + if (type & ompt_task_initial) { + ompt_data_t *parallel_data; + int team_size = 1; + ompt_get_parallel_info(0, ¶llel_data, &team_size); + ParallelData *PData = new ParallelData; + parallel_data->ptr = PData; + + Data = new TaskData(PData); + new_task_data->ptr = Data; + COUNT_EVENT2(task_create, initial); + } else if (type & ompt_task_undeferred) { + Data = new TaskData(ToTaskData(parent_task_data)); + new_task_data->ptr = Data; + Data->Included = true; + COUNT_EVENT2(task_create, included); + } else if (type & ompt_task_explicit || type & ompt_task_target) { + Data = new TaskData(ToTaskData(parent_task_data)); + new_task_data->ptr = Data; + + // Use the newly created address. We cannot use a single address from the + // parent because that would declare wrong relationships with other + // sibling tasks that may be created before this task is started! + TsanHappensBefore(Data->GetTaskPtr()); + ToTaskData(parent_task_data)->execution++; + COUNT_EVENT2(task_create, explicit); + } +} + +static void ompt_tsan_task_schedule(ompt_data_t *first_task_data, + ompt_task_status_t prior_task_status, + ompt_data_t *second_task_data) { + COUNT_EVENT1(task_schedule); + TaskData *FromTask = ToTaskData(first_task_data); + TaskData *ToTask = ToTaskData(second_task_data); + + if (ToTask->Included && prior_task_status != ompt_task_complete) + return; // No further synchronization for begin included tasks + if (FromTask->Included && prior_task_status == ompt_task_complete) { + // Just delete the task: + while (FromTask != nullptr && --FromTask->RefCount == 0) { + TaskData *Parent = FromTask->Parent; + if (FromTask->DependencyCount > 0) { + delete[] FromTask->Dependencies; + } + delete FromTask; + FromTask = Parent; + } + return; + } + + if (ToTask->execution == 0) { + ToTask->execution++; + // 1. Task will begin execution after it has been created. + TsanHappensAfter(ToTask->GetTaskPtr()); + for (unsigned i = 0; i < ToTask->DependencyCount; i++) { + ompt_dependence_t *Dependency = &ToTask->Dependencies[i]; + + TsanHappensAfter(Dependency->variable.ptr); + // in and inout dependencies are also blocked by prior in dependencies! + if (Dependency->dependence_type == ompt_dependence_type_out || Dependency->dependence_type == ompt_dependence_type_inout) { + TsanHappensAfter(ToInAddr(Dependency->variable.ptr)); + } + } + } else { + // 2. Task will resume after it has been switched away. + TsanHappensAfter(ToTask->GetTaskPtr()); + } + + if (prior_task_status != ompt_task_complete) { + ToTask->ImplicitTask = FromTask->ImplicitTask; + assert(ToTask->ImplicitTask != NULL && + "A task belongs to a team and has an implicit task on the stack"); + } + + // Task may be resumed at a later point in time. + TsanHappensBefore(FromTask->GetTaskPtr()); + + if (hasReductionCallback < ompt_set_always && FromTask->InBarrier) { + // We want to ignore writes in the runtime code during barriers, + // but not when executing tasks with user code! + TsanIgnoreWritesEnd(); + } + + if (prior_task_status == ompt_task_complete) { // task finished + + // Task will finish before a barrier in the surrounding parallel region ... + ParallelData *PData = FromTask->Team; + TsanHappensBefore( + PData->GetBarrierPtr(FromTask->ImplicitTask->BarrierIndex)); + + // ... and before an eventual taskwait by the parent thread. + TsanHappensBefore(FromTask->Parent->GetTaskwaitPtr()); + + if (FromTask->TaskGroup != nullptr) { + // This task is part of a taskgroup, so it will finish before the + // corresponding taskgroup_end. + TsanHappensBefore(FromTask->TaskGroup->GetPtr()); + } + for (unsigned i = 0; i < FromTask->DependencyCount; i++) { + ompt_dependence_t *Dependency = &FromTask->Dependencies[i]; + + // in dependencies block following inout and out dependencies! + TsanHappensBefore(ToInAddr(Dependency->variable.ptr)); + if (Dependency->dependence_type == ompt_dependence_type_out || Dependency->dependence_type == ompt_dependence_type_inout) { + TsanHappensBefore(Dependency->variable.ptr); + } + } + while (FromTask != nullptr && --FromTask->RefCount == 0) { + TaskData *Parent = FromTask->Parent; + if (FromTask->DependencyCount > 0) { + delete[] FromTask->Dependencies; + } + delete FromTask; + FromTask = Parent; + } + } + if (hasReductionCallback < ompt_set_always && ToTask->InBarrier) { + // We re-enter runtime code which currently performs a barrier. + TsanIgnoreWritesBegin(); + } +} + +static void ompt_tsan_dependences(ompt_data_t *task_data, + const ompt_dependence_t *deps, + int ndeps) { + COUNT_EVENT1(task_dependences); + if (ndeps > 0) { + // Copy the data to use it in task_switch and task_end. + TaskData *Data = ToTaskData(task_data); + Data->Dependencies = new ompt_dependence_t[ndeps]; + std::memcpy(Data->Dependencies, deps, + sizeof(ompt_dependence_t) * ndeps); + Data->DependencyCount = ndeps; + + // This callback is executed before this task is first started. + TsanHappensBefore(Data->GetTaskPtr()); + } +} + +/// OMPT event callbacks for handling locking. +static void ompt_tsan_mutex_acquired(ompt_mutex_t kind, + ompt_wait_id_t wait_id, + const void *codeptr_ra) { + if (archer_flags->print_ompt_counters) + switch (kind) { + case ompt_mutex_lock: + COUNT_EVENT2(mutex_acquired, lock); + break; + case ompt_mutex_nest_lock: + COUNT_EVENT2(mutex_acquired, nest_lock); + break; + case ompt_mutex_critical: + COUNT_EVENT2(mutex_acquired, critical); + break; + case ompt_mutex_atomic: + COUNT_EVENT2(mutex_acquired, atomic); + break; + case ompt_mutex_ordered: + COUNT_EVENT2(mutex_acquired, ordered); + break; + default: + COUNT_EVENT2(mutex_acquired, default); + break; + } + + // Acquire our own lock to make sure that + // 1. the previous release has finished. + // 2. the next acquire doesn't start before we have finished our release. + { + LocksMutex.lock(); + std::mutex &Lock = Locks[wait_id]; + LocksMutex.unlock(); + + Lock.lock(); + TsanHappensAfter(&Lock); + } +} + +static void ompt_tsan_mutex_released(ompt_mutex_t kind, + ompt_wait_id_t wait_id, + const void *codeptr_ra) { + if (archer_flags->print_ompt_counters) + switch (kind) { + case ompt_mutex_lock: + COUNT_EVENT2(mutex_released, lock); + break; + case ompt_mutex_nest_lock: + COUNT_EVENT2(mutex_released, nest_lock); + break; + case ompt_mutex_critical: + COUNT_EVENT2(mutex_released, critical); + break; + case ompt_mutex_atomic: + COUNT_EVENT2(mutex_released, atomic); + break; + case ompt_mutex_ordered: + COUNT_EVENT2(mutex_released, ordered); + break; + default: + COUNT_EVENT2(mutex_released, default); + break; + } + + { + LocksMutex.lock(); + std::mutex &Lock = Locks[wait_id]; + LocksMutex.unlock(); + TsanHappensBefore(&Lock); + + Lock.unlock(); + } +} + +// callback , signature , variable to store result , required support level +#define SET_OPTIONAL_CALLBACK_T(event, type, result, level) \ + do { \ + ompt_callback_##type##_t tsan_##event = &ompt_tsan_##event; \ + result = ompt_set_callback(ompt_callback_##event, \ + (ompt_callback_t)tsan_##event); \ + if (result < level) \ + printf("Registered callback '" #event "' is not supported at " #level " (%i)\n", \ + result); \ + } while (0) + +#define SET_CALLBACK_T(event, type) \ + do { \ + int res; \ + SET_OPTIONAL_CALLBACK_T(event, type, res, ompt_set_always); \ + } while (0) + +#define SET_CALLBACK(event) SET_CALLBACK_T(event, event) + +static int ompt_tsan_initialize(ompt_function_lookup_t lookup, + int device_num, + ompt_data_t *tool_data) { + const char *options = getenv("ARCHER_OPTIONS"); + archer_flags = new ArcherFlags(options); + + if (archer_flags->print_ompt_counters) + all_counter = new callback_counter_t[MAX_THREADS]; + + ompt_set_callback_t ompt_set_callback = + (ompt_set_callback_t)lookup("ompt_set_callback"); + if (ompt_set_callback == NULL) { + std::cerr << "Could not set callback, exiting..." << std::endl; + std::exit(1); + } + ompt_get_parallel_info = + (ompt_get_parallel_info_t)lookup("ompt_get_parallel_info"); + ompt_get_thread_data = (ompt_get_thread_data_t)lookup("ompt_get_thread_data"); + + if (ompt_get_parallel_info == NULL) { + fprintf(stderr, "Could not get inquiry function 'ompt_get_parallel_info', " + "exiting...\n"); + exit(1); + } + + SET_CALLBACK(thread_begin); + SET_CALLBACK(thread_end); + SET_CALLBACK(parallel_begin); + SET_CALLBACK(implicit_task); + SET_CALLBACK(sync_region); + SET_CALLBACK(parallel_end); + + SET_CALLBACK(task_create); + SET_CALLBACK(task_schedule); + SET_CALLBACK(dependences); + + SET_CALLBACK_T(mutex_acquired, mutex); + SET_CALLBACK_T(mutex_released, mutex); + SET_OPTIONAL_CALLBACK_T(reduction, sync_region, hasReductionCallback, ompt_set_never); + return 1; // success +} + +static void ompt_tsan_finalize(ompt_data_t *tool_data) { + if (archer_flags->print_ompt_counters) { + print_callbacks(all_counter); + delete[] all_counter; + } + + if (archer_flags->print_max_rss) { + struct rusage end; + getrusage(RUSAGE_SELF, &end); + printf("MAX RSS[KBytes] during execution: %ld\n", end.ru_maxrss); + } + + if (archer_flags) + delete archer_flags; +} + +extern "C" +ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version, + const char *runtime_version) { + const char *options = getenv("ARCHER_OPTIONS"); + archer_flags = new ArcherFlags(options); + if (!archer_flags->enabled) + { + if (archer_flags->verbose) + std::cout << "Archer disabled, stopping operation" + << std::endl; + delete archer_flags; + return NULL; + } + + static ompt_start_tool_result_t ompt_start_tool_result = { + &ompt_tsan_initialize, &ompt_tsan_finalize, {0}}; + runOnTsan=1; + RunningOnValgrind(); + if (!runOnTsan) // if we are not running on TSAN, give a different tool the + // chance to be loaded + { + if (archer_flags->verbose) + std::cout << "Archer detected OpenMP application without TSan " + "stopping operation" + << std::endl; + delete archer_flags; + return NULL; + } + + if (archer_flags->verbose) + std::cout << "Archer detected OpenMP application with TSan, supplying " + "OpenMP synchronization semantics" + << std::endl; + return &ompt_start_tool_result; +} Index: tools/archer/tests/CMakeLists.txt =================================================================== --- /dev/null +++ tools/archer/tests/CMakeLists.txt @@ -0,0 +1,37 @@ +# CMakeLists.txt file for unit testing Archer runtime library. +include(CheckFunctionExists) +include(CheckLibraryExists) + +# When using libgcc, -latomic may be needed for atomics +# (but when using compiler-rt, the atomics will be built-in) +# Note: we can not check for __atomic_load because clang treats it +# as special built-in and that breaks CMake checks +check_function_exists(__atomic_load_1 LIBARCHER_HAVE_BUILTIN_ATOMIC) +if(NOT LIBARCHER_HAVE_BUILTIN_ATOMIC) + check_library_exists(atomic __atomic_load_1 "" LIBARCHER_HAVE_LIBATOMIC) +else() + # not needed + set(LIBARCHER_HAVE_LIBATOMIC 0) +endif() + +set(LIBARCHER_TEST_PATH ${CMAKE_CURRENT_SOURCE_DIR}) +set(LIBARCHER_HAVE_ARCHER_LIBRARY ${LIBARCHER_LIB_PATH}) +set(LIBARCHER_HAVE_ARCHER_RUNTIME ${LIBARCHER_RUNTIME_PATH}) + +macro(pythonize_bool var) + if (${var}) + set(${var} True) + else() + set(${var} False) + endif() +endmacro() + +pythonize_bool(LIBARCHER_HAVE_ARCHER_LIBRARY) +pythonize_bool(LIBARCHER_HAVE_ARCHER_RUNTIME) +pythonize_bool(LIBARCHER_HAVE_LIBATOMIC) + +add_openmp_testsuite(check-libarcher "Running libarcher tests" ${CMAKE_CURRENT_BINARY_DIR} DEPENDS archer) + +# Configure the lit.site.cfg.in file +set(AUTO_GEN_COMMENT "## Autogenerated by libarcher configuration.\n# Do not edit!") +configure_file(lit.site.cfg.in lit.site.cfg @ONLY) Index: tools/archer/tests/barrier/barrier.c =================================================================== --- /dev/null +++ tools/archer/tests/barrier/barrier.c @@ -0,0 +1,41 @@ +/* + * barrier.c -- Archer testcase + */ + +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// +// See tools/archer/LICENSE.txt for details. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// RUN: %libarcher-compile-and-run | FileCheck %s +#include +#include + +int main(int argc, char *argv[]) { + int var = 0; + +#pragma omp parallel num_threads(2) shared(var) + { + if (omp_get_thread_num() == 0) { + var++; + } + +#pragma omp barrier + + if (omp_get_thread_num() == 1) { + var++; + } + } + + fprintf(stderr, "DONE\n"); + int error = (var != 2); + return error; +} + +// CHECK-NOT: ThreadSanitizer: data race +// CHECK-NOT: ThreadSanitizer: reported +// CHECK: DONE Index: tools/archer/tests/critical/critical.c =================================================================== --- /dev/null +++ tools/archer/tests/critical/critical.c @@ -0,0 +1,35 @@ +/* + * critical.c -- Archer testcase + */ + +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// +// See tools/archer/LICENSE.txt for details. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + + +// RUN: %libarcher-compile-and-run | FileCheck %s +#include +#include + +int main(int argc, char *argv[]) { + int var = 0; + +#pragma omp parallel num_threads(8) shared(var) + { +#pragma omp critical + { var++; } + } + + fprintf(stderr, "DONE\n"); + int error = (var != 8); + return error; +} + +// CHECK-NOT: ThreadSanitizer: data race +// CHECK-NOT: ThreadSanitizer: reported +// CHECK: DONE Index: tools/archer/tests/critical/lock-nested.c =================================================================== --- /dev/null +++ tools/archer/tests/critical/lock-nested.c @@ -0,0 +1,43 @@ +/* + * lock-nested.c -- Archer testcase + */ + +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// +// See tools/archer/LICENSE.txt for details. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + + +// RUN: %libarcher-compile-and-run | FileCheck %s +#include +#include + +int main(int argc, char *argv[]) { + int var = 0; + + omp_nest_lock_t lock; + omp_init_nest_lock(&lock); + +#pragma omp parallel num_threads(2) shared(var) + { + omp_set_nest_lock(&lock); + omp_set_nest_lock(&lock); + var++; + omp_unset_nest_lock(&lock); + omp_unset_nest_lock(&lock); + } + + omp_destroy_nest_lock(&lock); + + fprintf(stderr, "DONE\n"); + int error = (var != 2); + return error; +} + +// CHECK-NOT: ThreadSanitizer: data race +// CHECK-NOT: ThreadSanitizer: reported +// CHECK: DONE Index: tools/archer/tests/critical/lock.c =================================================================== --- /dev/null +++ tools/archer/tests/critical/lock.c @@ -0,0 +1,41 @@ +/* + * lock.c -- Archer testcase + */ + +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// +// See tools/archer/LICENSE.txt for details. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + + +// RUN: %libarcher-compile-and-run | FileCheck %s +#include +#include + +int main(int argc, char *argv[]) { + int var = 0; + + omp_lock_t lock; + omp_init_lock(&lock); + +#pragma omp parallel num_threads(2) shared(var) + { + omp_set_lock(&lock); + var++; + omp_unset_lock(&lock); + } + + omp_destroy_lock(&lock); + + fprintf(stderr, "DONE\n"); + int error = (var != 2); + return error; +} + +// CHECK-NOT: ThreadSanitizer: data race +// CHECK-NOT: ThreadSanitizer: reported +// CHECK: DONE Index: tools/archer/tests/deflake.bash =================================================================== --- /dev/null +++ tools/archer/tests/deflake.bash @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# This script is used to deflake inherently flaky archer tests. +# It is invoked from lit tests as: +# %deflake mybinary +# which is then substituted by lit to: +# $(dirname %s)/deflake.bash mybinary +# The script runs the target program up to 10 times, +# until it fails (i.e. produces a race report). + +for i in $(seq 1 10); do + OUT=`$@ 2>&1` + if [[ $? != 0 ]]; then + echo "$OUT" + exit 0 + fi +done +exit 1 Index: tools/archer/tests/lit.cfg =================================================================== --- /dev/null +++ tools/archer/tests/lit.cfg @@ -0,0 +1,135 @@ +# -*- Python -*- vim: set ft=python ts=4 sw=4 expandtab tw=79: +# Configuration file for the 'lit' test runner. + +import os +import re +import subprocess +import lit.formats + +# Tell pylint that we know config and lit_config exist somewhere. +if 'PYLINT_IMPORT' in os.environ: + config = object() + lit_config = object() + +def append_dynamic_library_path(path): + if config.operating_system == 'Windows': + name = 'PATH' + sep = ';' + elif config.operating_system == 'Darwin': + name = 'DYLD_LIBRARY_PATH' + sep = ':' + else: + name = 'LD_LIBRARY_PATH' + sep = ':' + if name in config.environment: + config.environment[name] = path + sep + config.environment[name] + else: + config.environment[name] = path + +# name: The name of this test suite. +config.name = 'libarcher' + +# suffixes: A list of file extensions to treat as test files. +config.suffixes = ['.c', '.cpp'] + +# test_source_root: The root path where tests are located. +config.test_source_root = os.path.dirname(__file__) + +# test_exec_root: The root object directory where output is placed +config.test_exec_root = config.libarcher_obj_root + +# test format +config.test_format = lit.formats.ShTest() + +# compiler flags +config.test_flags = " -I " + config.test_source_root + \ + " -I " + config.omp_header_dir + \ + " -L " + config.omp_library_dir + \ + " -Wl,-rpath," + config.omp_library_dir + \ + " " + config.test_extra_flags + +config.archer_flags = "-g -O1 -fsanitize=thread" + +libs_archer = "" +if config.has_archer_runtime: + libs_archer += " -L" + config.archer_runtime_dir + " -l" + \ + config.archer_runtime.replace("lib", "").replace(".so", "").replace(".dy", "") + \ + " -Wl,-rpath," + config.archer_runtime_dir + +# if config.has_archer_library: +# config.test_compiler += " -Xclang -load -Xclang " + \ +# config.archer_library_dir + "/" + config.archer_library + \ +# " -Wl,-rpath," + config.archer_library_dir + + +# extra libraries +libs = "" +if config.has_libatomic: + libs += " -latomic" + +# Allow XFAIL to work +config.target_triple = [ ] +for feature in config.test_compiler_features: + config.available_features.add(feature) + +# Setup environment to find dynamic library at runtime +append_dynamic_library_path(config.omp_library_dir) + +# Rpath modifications for Darwin +if config.operating_system == 'Darwin': + config.test_flags += " -Wl,-rpath," + config.omp_library_dir + +# Find the SDK on Darwin +if config.operating_system == 'Darwin': + cmd = subprocess.Popen(['xcrun', '--show-sdk-path'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = cmd.communicate() + out = out.strip() + res = cmd.wait() + if res == 0 and out: + config.test_flags += " -isysroot " + out + +config.suppression = "" +# need to deal with the static analysis with another execution line in the test +# if not config.has_archer_library and config.has_archer_runtime: +if config.has_archer_runtime: + config.suppression = "env TSAN_OPTIONS=\"ignore_noninstrumented_modules=1\"" + +if 'Linux' in config.operating_system: + config.available_features.add("linux") + +# to run with icc INTEL_LICENSE_FILE must be set +if 'INTEL_LICENSE_FILE' in os.environ: + config.environment['INTEL_LICENSE_FILE'] = os.environ['INTEL_LICENSE_FILE'] + +# Race Tests +config.substitutions.append(("%libarcher-compile-and-run-race", \ + "%libarcher-compile && %libarcher-run-race")) +config.substitutions.append(("%libarcher-compile-and-run", \ + "%libarcher-compile && %libarcher-run")) +config.substitutions.append(("%libarcher-cxx-compile-and-run", \ + "%libarcher-cxx-compile && %libarcher-run")) +config.substitutions.append(("%libarcher-cxx-compile", \ + "%clang-archerXX %openmp_flags %archer_flags %flags -std=c++11 %s -o %t" + libs)) +config.substitutions.append(("%libarcher-compile", \ + "%clang-archer %openmp_flags %archer_flags %flags %s -o %t" + libs + libs_archer)) +config.substitutions.append(("%libarcher-run-race", "%suppression %deflake %t 2>&1 | tee %t.log")) +config.substitutions.append(("%libarcher-run", "%suppression %t 2>&1 | tee %t.log")) +config.substitutions.append(("%clang-archerXX", config.test_cxx_compiler)) +config.substitutions.append(("%clang-archer", config.test_c_compiler)) +config.substitutions.append(("%openmp_flags", config.test_openmp_flags)) +config.substitutions.append(("%archer_flags", config.archer_flags)) +config.substitutions.append(("%flags", config.test_flags)) +config.substitutions.append(("%suppression", "env OMP_TOOL_LIBRARIES=" + config.libomp_obj_root + \ + "/../libarcher.so TSAN_OPTIONS='ignore_noninstrumented_modules=1'")) +config.substitutions.append(("%deflake", os.path.join(os.path.dirname(__file__), "deflake.bash"))) + +config.substitutions.append(("FileCheck", config.test_filecheck)) +config.substitutions.append(("%sort-threads", "sort --numeric-sort --stable")) +if config.operating_system == 'Windows': + # No such environment variable on Windows. + config.substitutions.append(("%preload-tool", "true ||")) +elif config.operating_system == 'Darwin': + config.substitutions.append(("%preload-tool", "env DYLD_INSERT_LIBRARIES=%T/tool.so")) +else: + config.substitutions.append(("%preload-tool", "env LD_PRELOAD=%T/tool.so")) Index: tools/archer/tests/lit.site.cfg.in =================================================================== --- /dev/null +++ tools/archer/tests/lit.site.cfg.in @@ -0,0 +1,27 @@ +@AUTO_GEN_COMMENT@ + +config.test_c_compiler = "@OPENMP_TEST_C_COMPILER@" +config.test_cxx_compiler = "@OPENMP_TEST_CXX_COMPILER@" +config.test_compiler_features = @OPENMP_TEST_COMPILER_FEATURES@ +config.test_filecheck = "@OPENMP_FILECHECK_EXECUTABLE@" +config.test_openmp_flags = "@OPENMP_TEST_OPENMP_FLAGS@" +config.test_extra_flags = "@OPENMP_TEST_FLAGS@" +config.libomp_obj_root = "@CMAKE_CURRENT_BINARY_DIR@" +config.omp_library_dir = "@LIBOMP_LIBRARY_DIR@" +config.omp_header_dir = "@LIBOMP_INCLUDE_DIR@" +config.operating_system = "@CMAKE_SYSTEM_NAME@" +config.has_libatomic = @LIBOMP_HAVE_LIBATOMIC@ + +config.test_archer_flags = "@OPENMP_TEST_ARCHER_FLAGS@" +config.libarcher_obj_root = "@CMAKE_CURRENT_BINARY_DIR@" + +config.archer_tools_dir = "@LIBARCHER_TOOLS_DIR@" +config.archer_library_dir = "@LIBARCHER_LIB_PATH@" +config.archer_runtime_dir = "@LIBARCHER_RUNTIME_PATH@" +config.archer_library = "@LIBARCHER_LIB@" +config.archer_runtime = "@LIBARCHER_RTL@" +config.has_archer_library = @LIBARCHER_HAVE_ARCHER_LIBRARY@ +config.has_archer_runtime = @LIBARCHER_HAVE_ARCHER_RUNTIME@ + +# Let the main config do the real work. +lit_config.load_config(config, "@CMAKE_CURRENT_SOURCE_DIR@/lit.cfg") Index: tools/archer/tests/ompt/ompt-signal.h =================================================================== --- /dev/null +++ tools/archer/tests/ompt/ompt-signal.h @@ -0,0 +1,42 @@ +/* + * ompt-signal.h -- Header providing low-level synchronization for tests + */ + +//===----------------------------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is a copy from runtime/test/ompt/ +// +//===----------------------------------------------------------------------===// + +#if defined(WIN32) || defined(_WIN32) +#include +#define delay() Sleep(1); +#else +#include +#define delay(t) usleep(t); +#endif + +// These functions are used to provide a signal-wait mechanism to enforce +// expected scheduling for the test cases. +// Conditional variable (s) needs to be shared! Initialize to 0 + +#define OMPT_SIGNAL(s) ompt_signal(&s) +// inline +void ompt_signal(int *s) { +#pragma omp atomic + (*s)++; +} + +#define OMPT_WAIT(s, v) ompt_wait(&s, v) +// wait for s >= v +// inline +void ompt_wait(int *s, int v) { + int wait = 0; + do { + delay(10); +#pragma omp atomic read + wait = (*s); + } while (wait < v); +} Index: tools/archer/tests/parallel/parallel-firstprivate.c =================================================================== --- /dev/null +++ tools/archer/tests/parallel/parallel-firstprivate.c @@ -0,0 +1,32 @@ +/* + * parallel-firstprivate.c -- Archer testcase + */ + +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// +// See tools/archer/LICENSE.txt for details. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + + +// RUN: %libarcher-compile-and-run | FileCheck %s +#include +#include + +int main(int argc, char *argv[]) { + int var = 0; + +#pragma omp parallel num_threads(2) firstprivate(var) + { var = 1; } + + fprintf(stderr, "DONE\n"); + // var should still be 0! + return var; +} + +// CHECK-NOT: ThreadSanitizer: data race +// CHECK-NOT: ThreadSanitizer: reported +// CHECK: DONE Index: tools/archer/tests/parallel/parallel-simple.c =================================================================== --- /dev/null +++ tools/archer/tests/parallel/parallel-simple.c @@ -0,0 +1,38 @@ +/* + * parallel-simple.c -- Archer testcase + */ + +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// +// See tools/archer/LICENSE.txt for details. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + + +// RUN: %libarcher-compile-and-run | FileCheck %s +#include +#include + +int main(int argc, char *argv[]) { + int var = 0; + +#pragma omp parallel num_threads(2) shared(var) + { + if (omp_get_thread_num() == 1) { + var++; + } + } // implicit barrier + + var++; + + fprintf(stderr, "DONE\n"); + int error = (var != 2); + return error; +} + +// CHECK-NOT: ThreadSanitizer: data race +// CHECK-NOT: ThreadSanitizer: reported +// CHECK: DONE Index: tools/archer/tests/parallel/parallel-simple2.c =================================================================== --- /dev/null +++ tools/archer/tests/parallel/parallel-simple2.c @@ -0,0 +1,43 @@ +/* + * parallel-simple2.c -- Archer testcase + */ + +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// +// See tools/archer/LICENSE.txt for details. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + + +// RUN: %libarcher-compile-and-run | FileCheck %s +#include +#include + +int main(int argc, char *argv[]) { + int var = 0; + +// Create team of threads so that there is no implicit happens before +// when creating the thread. +#pragma omp parallel num_threads(2) + {} + + var++; + +#pragma omp parallel num_threads(2) shared(var) + { + if (omp_get_thread_num() == 1) { + var++; + } + } // implicit barrier + + fprintf(stderr, "DONE\n"); + int error = (var != 2); + return error; +} + +// CHECK-NOT: ThreadSanitizer: data race +// CHECK-NOT: ThreadSanitizer: reported +// CHECK: DONE Index: tools/archer/tests/races/critical-unrelated.c =================================================================== --- /dev/null +++ tools/archer/tests/races/critical-unrelated.c @@ -0,0 +1,42 @@ +/* + * critical-unrelated.c -- Archer testcase + */ + +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// +// See tools/archer/LICENSE.txt for details. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + + +// RUN: %libarcher-compile-and-run-race | FileCheck %s +#include +#include + +int main(int argc, char *argv[]) { + int var = 0; + +#pragma omp parallel num_threads(2) shared(var) + { +#pragma omp critical + { + // Dummy region. + } + + var++; + } + + fprintf(stderr, "DONE\n"); +} + +// CHECK: WARNING: ThreadSanitizer: data race +// CHECK-NEXT: {{(Write|Read)}} of size 4 +// CHECK-NEXT: #0 {{.*}}critical-unrelated.c:29 +// CHECK: Previous write of size 4 +// CHECK-NEXT: #0 {{.*}}critical-unrelated.c:29 +// CHECK: DONE +// CHECK: ThreadSanitizer: reported 1 warnings + Index: tools/archer/tests/races/lock-nested-unrelated.c =================================================================== --- /dev/null +++ tools/archer/tests/races/lock-nested-unrelated.c @@ -0,0 +1,48 @@ +/* + * lock-nested-unrelated.c -- Archer testcase + */ + +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// +// See tools/archer/LICENSE.txt for details. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + + +// RUN: %libarcher-compile-and-run-race | FileCheck %s +#include +#include + +int main(int argc, char *argv[]) { + int var = 0; + + omp_nest_lock_t lock; + omp_init_nest_lock(&lock); + +#pragma omp parallel num_threads(2) shared(var) + { + omp_set_nest_lock(&lock); + omp_set_nest_lock(&lock); + // Dummy locking. + omp_unset_nest_lock(&lock); + omp_unset_nest_lock(&lock); + + var++; + } + + omp_destroy_nest_lock(&lock); + + fprintf(stderr, "DONE\n"); +} + +// CHECK: WARNING: ThreadSanitizer: data race +// CHECK-NEXT: {{(Write|Read)}} of size 4 +// CHECK-NEXT: #0 {{.*}}lock-nested-unrelated.c:33 +// CHECK: Previous write of size 4 +// CHECK-NEXT: #0 {{.*}}lock-nested-unrelated.c:33 +// CHECK: DONE +// CHECK: ThreadSanitizer: reported 1 warnings + Index: tools/archer/tests/races/lock-unrelated.c =================================================================== --- /dev/null +++ tools/archer/tests/races/lock-unrelated.c @@ -0,0 +1,48 @@ +/* + * lock-unrelated.c -- Archer testcase + */ + +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// +// See tools/archer/LICENSE.txt for details. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + + +// RUN: %libarcher-compile-and-run-race | FileCheck %s +#include +#include + +int main(int argc, char *argv[]) { + int var = 0; + + omp_lock_t lock; + omp_init_lock(&lock); + +#pragma omp parallel num_threads(2) shared(var) + { + omp_set_lock(&lock); + // Dummy locking. + omp_unset_lock(&lock); + + var++; + } + + omp_destroy_lock(&lock); + + int error = (var != 2); + fprintf(stderr, "DONE\n"); + return error; +} + +// CHECK: WARNING: ThreadSanitizer: data race +// CHECK-NEXT: {{(Write|Read)}} of size 4 +// CHECK-NEXT: #0 {{.*}}lock-unrelated.c:31 +// CHECK: Previous write of size 4 +// CHECK-NEXT: #0 {{.*}}lock-unrelated.c:31 +// CHECK: DONE +// CHECK: ThreadSanitizer: reported 1 warnings + Index: tools/archer/tests/races/parallel-simple.c =================================================================== --- /dev/null +++ tools/archer/tests/races/parallel-simple.c @@ -0,0 +1,37 @@ +/* + * parallel-simple.c -- Archer testcase + */ + +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// +// See tools/archer/LICENSE.txt for details. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + + +// RUN: %libarcher-compile-and-run-race | FileCheck %s +#include +#include + +int main(int argc, char *argv[]) { + int var = 0; + +#pragma omp parallel num_threads(2) shared(var) + { var++; } + + int error = (var != 2); + fprintf(stderr, "DONE\n"); + return error; +} + +// CHECK: WARNING: ThreadSanitizer: data race +// CHECK-NEXT: {{(Write|Read)}} of size 4 +// CHECK-NEXT: #0 {{.*}}parallel-simple.c:23 +// CHECK: Previous write of size 4 +// CHECK-NEXT: #0 {{.*}}parallel-simple.c:23 +// CHECK: DONE +// CHECK: ThreadSanitizer: reported 1 warnings + Index: tools/archer/tests/races/task-dependency.c =================================================================== --- /dev/null +++ tools/archer/tests/races/task-dependency.c @@ -0,0 +1,61 @@ +/* + * task-deoendency.c -- Archer testcase + */ + +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// +// See tools/archer/LICENSE.txt for details. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + + +// RUN: %libarcher-compile-and-run-race | FileCheck %s +#include +#include +#include +#include "ompt/ompt-signal.h" + +int main(int argc, char *argv[]) { + int var = 0, a = 0; + +#pragma omp parallel num_threads(2) shared(var, a) +#pragma omp master + { +#pragma omp task shared(var, a) depend(out : var) + { + OMPT_SIGNAL(a); + var++; + } + +#pragma omp task shared(a) depend(in : var) + { + OMPT_SIGNAL(a); + OMPT_WAIT(a, 3); + } + +#pragma omp task shared(var) // depend(in: var) is missing here! + { + var++; + OMPT_SIGNAL(a); + } + + // Give other thread time to steal the task. + OMPT_WAIT(a, 2); + } + + int error = (var != 2); + fprintf(stderr, "DONE\n"); + return error; +} + +// CHECK: WARNING: ThreadSanitizer: data race +// CHECK-NEXT: {{(Write|Read)}} of size 4 +// CHECK-NEXT: #0 {{.*}}task-dependency.c:41 +// CHECK: Previous write of size 4 +// CHECK-NEXT: #0 {{.*}}task-dependency.c:30 +// CHECK: DONE +// CHECK: ThreadSanitizer: reported 1 warnings + Index: tools/archer/tests/races/task-taskgroup-unrelated.c =================================================================== --- /dev/null +++ tools/archer/tests/races/task-taskgroup-unrelated.c @@ -0,0 +1,61 @@ +/* + * task-taskgroup-unrelated.c -- Archer testcase + */ + +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// +// See tools/archer/LICENSE.txt for details. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// RUN: %libarcher-compile-and-run-race | FileCheck %s +#include +#include +#include +#include "ompt/ompt-signal.h" + +int main(int argc, char *argv[]) { + int var = 0, a = 0; + +#pragma omp parallel num_threads(2) shared(var, a) +#pragma omp master + { +#pragma omp task shared(var, a) + { + var++; + OMPT_SIGNAL(a); + // Give master thread time to execute the task in the taskgroup. + OMPT_WAIT(a, 2); + } + +#pragma omp taskgroup + { +#pragma omp task if (0) + { + // Dummy task. + } + + // Give other threads time to steal the tasks. + OMPT_WAIT(a, 1); + OMPT_SIGNAL(a); + } + + var++; + } + + int error = (var != 2); + fprintf(stderr, "DONE\n"); + return error; +} + +// CHECK: WARNING: ThreadSanitizer: data race +// CHECK-NEXT: {{(Write|Read)}} of size 4 +// CHECK-NEXT: #0 {{.*}}task-taskgroup-unrelated.c:46 +// CHECK: Previous write of size 4 +// CHECK-NEXT: #0 {{.*}}task-taskgroup-unrelated.c:28 +// CHECK: DONE +// CHECK: ThreadSanitizer: reported 1 warnings + Index: tools/archer/tests/races/task-taskwait-nested.c =================================================================== --- /dev/null +++ tools/archer/tests/races/task-taskwait-nested.c @@ -0,0 +1,59 @@ +/* + * task-taskwait-nested.c -- Archer testcase + */ + +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// +// See tools/archer/LICENSE.txt for details. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + + +// RUN: %libarcher-compile-and-run-race | FileCheck %s +#include +#include +#include +#include "ompt/ompt-signal.h" + +int main(int argc, char *argv[]) { + int var = 0, a = 0; + +#pragma omp parallel num_threads(2) shared(var, a) +#pragma omp master + { +#pragma omp task shared(var, a) + { +#pragma omp task shared(var, a) + { + // wait for master to pass the taskwait + OMPT_SIGNAL(a); + OMPT_WAIT(a, 2); + var++; + } + } + + // Give other thread time to steal the task and execute its child. + OMPT_WAIT(a, 1); + +// Only directly generated children are guaranteed to be executed. +#pragma omp taskwait + OMPT_SIGNAL(a); + var++; + } + + int error = (var != 2); + fprintf(stderr, "DONE\n"); + return error; +} + +// CHECK: WARNING: ThreadSanitizer: data race +// CHECK-NEXT: {{(Write|Read)}} of size 4 +// CHECK-NEXT: #0 {{.*}}task-taskwait-nested.c:34 +// CHECK: Previous write of size 4 +// CHECK-NEXT: #0 {{.*}}task-taskwait-nested.c:44 +// CHECK: DONE +// CHECK: ThreadSanitizer: reported 1 warnings + Index: tools/archer/tests/races/task-two.c =================================================================== --- /dev/null +++ tools/archer/tests/races/task-two.c @@ -0,0 +1,45 @@ +/* + * task-two.c -- Archer testcase + */ + +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// +// See tools/archer/LICENSE.txt for details. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + + +// RUN: %libarcher-compile-and-run-race | FileCheck %s +#include +#include +#include + +#define NUM_THREADS 2 + +int main(int argc, char *argv[]) { + int var = 0; + int i; + +#pragma omp parallel for num_threads(NUM_THREADS) shared(var) schedule(static, \ + 1) + for (i = 0; i < NUM_THREADS; i++) { +#pragma omp task shared(var) if (0) // the task is inlined an executed locally + { var++; } + } + + int error = (var != 2); + fprintf(stderr, "DONE\n"); + return error; +} + +// CHECK: WARNING: ThreadSanitizer: data race +// CHECK-NEXT: {{(Write|Read)}} of size 4 +// CHECK-NEXT: #0 {{.*}}task-two.c:30 +// CHECK: Previous write of size 4 +// CHECK-NEXT: #0 {{.*}}task-two.c:30 +// CHECK: DONE +// CHECK: ThreadSanitizer: reported 1 warnings + Index: tools/archer/tests/reduction/parallel-reduction-nowait.c =================================================================== --- /dev/null +++ tools/archer/tests/reduction/parallel-reduction-nowait.c @@ -0,0 +1,45 @@ +/* + * parallel-reduction-nowait.c -- Archer testcase + */ + +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// +// See tools/archer/LICENSE.txt for details. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + + +// RUN: %libarcher-compile-and-run | FileCheck %s +#include +#include + +int main(int argc, char *argv[]) { + int var = 0, i; + int sum1 = 0; + int sum2 = 0; + +// Number of threads is empirical: We need enough threads so that +// the reduction is really performed hierarchically in the barrier! +#pragma omp parallel num_threads(5) reduction(+ : var) + { +#pragma omp for schedule(static) nowait reduction(+ : sum1) + for (i = 0; i < 5; i++) + sum1 += i; +#pragma omp for schedule(static) reduction(+ : sum2) + for (i = 0; i < 5; i++) + sum2 += i; + + var = sum1 + sum2; + } + + fprintf(stderr, "DONE\n"); + int error = (var != 100); + return error; +} + +// CHECK-NOT: ThreadSanitizer: data race +// CHECK-NOT: ThreadSanitizer: reported +// CHECK: DONE Index: tools/archer/tests/reduction/parallel-reduction.c =================================================================== --- /dev/null +++ tools/archer/tests/reduction/parallel-reduction.c @@ -0,0 +1,34 @@ +/* + * parallel-reduction.c -- Archer testcase + */ + +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// +// See tools/archer/LICENSE.txt for details. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + + +// RUN: %libarcher-compile-and-run| FileCheck %s +#include +#include + +int main(int argc, char *argv[]) { + int var = 0; + +// Number of threads is empirical: We need enough threads so that +// the reduction is really performed hierarchically in the barrier! +#pragma omp parallel num_threads(5) reduction(+ : var) + { var = 1; } + + fprintf(stderr, "DONE\n"); + int error = (var != 5); + return error; +} + +// CHECK-NOT: ThreadSanitizer: data race +// CHECK-NOT: ThreadSanitizer: reported +// CHECK: DONE Index: tools/archer/tests/task/task-barrier.c =================================================================== --- /dev/null +++ tools/archer/tests/task/task-barrier.c @@ -0,0 +1,51 @@ +/* + * task-barrier.c -- Archer testcase + */ + +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// +// See tools/archer/LICENSE.txt for details. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + + +// RUN: %libarcher-compile-and-run | FileCheck %s +#include +#include +#include +#include "ompt/ompt-signal.h" + +int main(int argc, char *argv[]) { + int var = 0, a = 0; + +#pragma omp parallel num_threads(2) shared(var, a) + { +#pragma omp master + { +#pragma omp task shared(var) + { + OMPT_SIGNAL(a); + var++; + } + + // Give other thread time to steal the task. + OMPT_WAIT(a, 1); + } + +#pragma omp barrier + +#pragma omp master + { var++; } + } + + fprintf(stderr, "DONE\n"); + int error = (var != 2); + return error; +} + +// CHECK-NOT: ThreadSanitizer: data race +// CHECK-NOT: ThreadSanitizer: reported +// CHECK: DONE Index: tools/archer/tests/task/task-create.c =================================================================== --- /dev/null +++ tools/archer/tests/task/task-create.c @@ -0,0 +1,45 @@ +/* + * task-create.c -- Archer testcase + */ + +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// +// See tools/archer/LICENSE.txt for details. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + + +// RUN: %libarcher-compile-and-run | FileCheck %s +#include +#include +#include +#include "ompt/ompt-signal.h" + +int main(int argc, char *argv[]) { + int var = 0, a = 0; + +#pragma omp parallel num_threads(2) shared(var, a) +#pragma omp master + { + var++; +#pragma omp task shared(var, a) + { + var++; + OMPT_SIGNAL(a); + } + + // Give other thread time to steal the task. + OMPT_WAIT(a, 1); + } + + fprintf(stderr, "DONE\n"); + int error = (var != 2); + return error; +} + +// CHECK-NOT: ThreadSanitizer: data race +// CHECK-NOT: ThreadSanitizer: reported +// CHECK: DONE Index: tools/archer/tests/task/task-dependency.c =================================================================== --- /dev/null +++ tools/archer/tests/task/task-dependency.c @@ -0,0 +1,53 @@ +/* + * task-dependency.c -- Archer testcase + */ + +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// +// See tools/archer/LICENSE.txt for details. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + + +// RUN: %libarcher-compile-and-run | FileCheck %s +#include +#include +#include +#include "ompt/ompt-signal.h" + +int main(int argc, char *argv[]) { + int var = 0, a = 0; + +#pragma omp parallel num_threads(2) shared(var, a) +#pragma omp master + { +#pragma omp task shared(var, a) depend(out : var) + { + var++; + OMPT_SIGNAL(a); + } + +#pragma omp task shared(var, a) depend(in : var) + { OMPT_WAIT(a, 2); } + +#pragma omp task shared(var, a) depend(in : var) + { + OMPT_SIGNAL(a); + var++; + } + + // Give other thread time to steal the task. + OMPT_WAIT(a, 1); + } + + fprintf(stderr, "DONE\n"); + int error = (var != 2); + return error; +} + +// CHECK-NOT: ThreadSanitizer: data race +// CHECK-NOT: ThreadSanitizer: reported +// CHECK: DONE Index: tools/archer/tests/task/task-taskgroup-nested.c =================================================================== --- /dev/null +++ tools/archer/tests/task/task-taskgroup-nested.c @@ -0,0 +1,52 @@ +/* + * task-taskgroup-nested.c -- Archer testcase + */ + +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// +// See tools/archer/LICENSE.txt for details. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + + +// RUN: %libarcher-compile-and-run | FileCheck %s +#include +#include +#include +#include "ompt/ompt-signal.h" + +int main(int argc, char *argv[]) { + int var = 0, a = 0; + +#pragma omp parallel num_threads(2) shared(var, a) +#pragma omp master + { +#pragma omp taskgroup + { +#pragma omp task + { +#pragma omp task shared(var, a) + { + var++; + OMPT_SIGNAL(a); + } + } + + // Give other thread time to steal the task and execute its child. + OMPT_WAIT(a, 1); + } + + var++; + } + + fprintf(stderr, "DONE\n"); + int error = (var != 2); + return error; +} + +// CHECK-NOT: ThreadSanitizer: data race +// CHECK-NOT: ThreadSanitizer: reported +// CHECK: DONE Index: tools/archer/tests/task/task-taskgroup.c =================================================================== --- /dev/null +++ tools/archer/tests/task/task-taskgroup.c @@ -0,0 +1,49 @@ +/* + * task-taskgroup.c -- Archer testcase + */ + +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// +// See tools/archer/LICENSE.txt for details. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + + +// RUN: %libarcher-compile-and-run | FileCheck %s +#include +#include +#include +#include "ompt/ompt-signal.h" + +int main(int argc, char *argv[]) { + int var = 0, a = 0; + +#pragma omp parallel num_threads(2) shared(var, a) +#pragma omp master + { +#pragma omp taskgroup + { +#pragma omp task shared(var, a) + { + var++; + OMPT_SIGNAL(a); + } + + // Give other thread time to steal the task. + OMPT_WAIT(a, 1); + } + + var++; + } + + fprintf(stderr, "DONE\n"); + int error = (var != 2); + return error; +} + +// CHECK-NOT: ThreadSanitizer: data race +// CHECK-NOT: ThreadSanitizer: reported +// CHECK: DONE Index: tools/archer/tests/task/task-taskwait-nested.c =================================================================== --- /dev/null +++ tools/archer/tests/task/task-taskwait-nested.c @@ -0,0 +1,52 @@ +/* + * task-taskwait-nested.c -- Archer testcase + */ + +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// +// See tools/archer/LICENSE.txt for details. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + + +// RUN: %libarcher-compile-and-run | FileCheck %s +#include +#include +#include +#include "ompt/ompt-signal.h" + +int main(int argc, char *argv[]) { + int var = 0, a = 0; + +#pragma omp parallel num_threads(2) shared(var, a) +#pragma omp master + { +#pragma omp task + { +#pragma omp task shared(var, a) + { + OMPT_SIGNAL(a); + delay(100); + var++; + } +#pragma omp taskwait + } + + // Give other thread time to steal the task and execute its child. + OMPT_WAIT(a, 1); + +#pragma omp taskwait + var++; + } + + fprintf(stderr, "DONE\n"); + int error = (var != 2); + return error; +} + +// CHECK-NOT: ThreadSanitizer: data race +// CHECK-NOT: ThreadSanitizer: reported +// CHECK: DONE Index: tools/archer/tests/task/task-taskwait.c =================================================================== --- /dev/null +++ tools/archer/tests/task/task-taskwait.c @@ -0,0 +1,49 @@ +/* + * task-taskwait.c -- Archer testcase + */ + +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// +// See tools/archer/LICENSE.txt for details. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + + +// RUN: %libarcher-compile-and-run | FileCheck %s +#include +#include +#include +#include "ompt/ompt-signal.h" + +int main(int argc, char *argv[]) { + int var = 0, a = 0; + +#pragma omp parallel num_threads(2) shared(var, a) +#pragma omp master + { +#pragma omp task shared(var, a) + { + OMPT_SIGNAL(a); + OMPT_WAIT(a, 2); + delay(100); + var++; + } + + // Give other thread time to steal the task. + OMPT_WAIT(a, 1); + OMPT_SIGNAL(a); +#pragma omp taskwait + var++; + } + + fprintf(stderr, "DONE\n"); + int error = (var != 2); + return error; +} + +// CHECK-NOT: ThreadSanitizer: data race +// CHECK-NOT: ThreadSanitizer: reported +// CHECK: DONE Index: tools/archer/tests/worksharing/ordered.c =================================================================== --- /dev/null +++ tools/archer/tests/worksharing/ordered.c @@ -0,0 +1,38 @@ +/* + * ordered.c -- Archer testcase + */ + +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// +// See tools/archer/LICENSE.txt for details. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + + +// RUN: %libarcher-compile-and-run | FileCheck %s +#include +#include + +#define NUM_THREADS 2 + +int main(int argc, char *argv[]) { + int var = 0; + int i; + +#pragma omp parallel for ordered num_threads(NUM_THREADS) shared(var) + for (i = 0; i < NUM_THREADS; i++) { +#pragma omp ordered + { var++; } + } + + fprintf(stderr, "DONE\n"); + int error = (var != 2); + return error; +} + +// CHECK-NOT: ThreadSanitizer: data race +// CHECK-NOT: ThreadSanitizer: reported +// CHECK: DONE