[OpenMP][Tool] Add Archer option to disable data race analysis for sequential part

[protze.joachim]

Summary

This introduces the new ARCHER_OPTIONS flag ignore_serial=0|1 to disable analysis and logging of memory accesses in the sequential part of the OpenMP application.

In the sequential part of an OpenMP program no data race is possible, unless there is non-OpenMP concurrency (such as pthreads, MPI, ...). For the latter reason, this is not active by default.

Besides reducing the runtime overhead for the sequential part of the program, this reduces the memory overhead for sequential initialization. In combination with flush_shadow=1 this can allow analysis of applications, which run close to the limit of available memory, but only access smaller parts of shared memory during each OpenMP parallel region.

The problem is that Archer only gets active, when the OpenMP runtime gets initialized, which might be after serial initialization of the application. In such case, it helps to call for example omp_get_max_threads() at the beginning of main.

[jdoerfert]

Can we somehow test this?

[protze.joachim]

I added run lines to the tests with race, to verify that the races are still detected, if ignore_serial=1 .

openmp/tools/archer/README.md

 <th scope="col" class="org-left">Description</th>
 </tr>
 </thead>
 
 <tbody>
 <tr>
 <td class="org-left">flush&#95;shadow</td>
 <td class="org-right">0</td>
-<td class="org-left">Flush shadow memory at the end of an outer OpenMP parallel region. Our experiments show that this can reduce memory overhead by ~30% and runtime overhead by ~10%. This flag is useful for large OpenMP applications that typically require large amounts of memory, causing out-of-memory exceptions when checked by Archer.</td>
+<td class="org-left">Flush shadow memory at the end of an outer OpenMP
+parallel region. Our experiments show that this can reduce memory overhead
+by ~30% and runtime overhead by ~10%. This flag is useful for large OpenMP
+applications that typically require large amounts of memory, causing
+out-of-memory exceptions when checked by Archer.</td>
 </tr>
 </tbody>
 
 <tbody>
 <tr>
 <td class="org-left">print&#95;max&#95;rss</td>
 <td class="org-right">0</td>
 <td class="org-left">Print the RSS memory peak at the end of the execution.</td>
 </tr>
 </tbody>
 
 <tbody>
 <tr>
+<td class="org-left">ignore&#95;serial</td>
+<td class="org-right">0</td>
+<td class="org-left">Turn off tracking and analysis of memory accesses in
+the sequential part of an OpenMP program. (Only effective when OpenMP
+runtime is initialized. In doubt, insert omp_get_max_threads() as first
+statement in main!)</td>
+</tr>
+</tbody>
+
+<tbody>
+<tr>
 <td class="org-left">verbose</td>
 <td class="org-right">0</td>
 <td class="org-left">Print startup information.</td>
 </tr>
 </tbody>
 
 <tbody>
 <tr>

openmp/tools/archer/ompt-tsan.cpp

 #include "omp-tools.h"
 
 static int runOnTsan;
 static int hasReductionCallback;
 
 class ArcherFlags {
 public:
 #if (LLVM_VERSION) >= 40
-  int flush_shadow;
+  int flush_shadow{0};
 #endif
-  int print_max_rss;
-  int verbose;
-  int enabled;
+  int print_max_rss{0};
+  int verbose{0};
+  int enabled{1};
+  int ignore_serial{0};
 
-  ArcherFlags(const char *env)
-      :
-#if (LLVM_VERSION) >= 40
-        flush_shadow(0),
-#endif
-        print_max_rss(0), verbose(0), enabled(1) {
+  ArcherFlags(const char *env) {
     if (env) {
       std::vector<std::string> tokens;
       std::string token;
       std::string str(env);
       std::istringstream iss(str);
       while (std::getline(iss, token, ' '))
         tokens.push_back(token);
 
       for (std::vector<std::string>::iterator it = tokens.begin();
            it != tokens.end(); ++it) {
 #if (LLVM_VERSION) >= 40
         if (sscanf(it->c_str(), "flush_shadow=%d", &flush_shadow))
           continue;
 #endif
         if (sscanf(it->c_str(), "print_max_rss=%d", &print_max_rss))
           continue;
         if (sscanf(it->c_str(), "verbose=%d", &verbose))
           continue;
         if (sscanf(it->c_str(), "enable=%d", &enabled))
           continue;
+        if (sscanf(it->c_str(), "ignore_serial=%d", &ignore_serial))
+          continue;
         std::cerr << "Illegal values for ARCHER_OPTIONS variable: " << token
                   << std::endl;
       }
     }
   }
 };
 
 class TsanFlags {
static void ompt_tsan_parallel_begin(ompt_data_t *parent_task_data,
                                      ompt_data_t *parallel_data,
                                      uint32_t requested_team_size,
                                      int flag,
                                      const void *codeptr_ra) {
   ParallelData *Data = new ParallelData(codeptr_ra);
   parallel_data->ptr = Data;
 
   TsanHappensBefore(Data->GetParallelPtr());
+  if (archer_flags->ignore_serial && ToTaskData(parent_task_data)->isInitial())
+    TsanIgnoreWritesEnd();
 }
 
 static void ompt_tsan_parallel_end(ompt_data_t *parallel_data,
                                    ompt_data_t *task_data,
                                    int flag,
                                    const void *codeptr_ra) {
+  if (archer_flags->ignore_serial && ToTaskData(task_data)->isInitial())
+    TsanIgnoreWritesBegin();
   ParallelData *Data = ToParallelData(parallel_data);
   TsanHappensAfter(Data->GetBarrierPtr(0));
   TsanHappensAfter(Data->GetBarrierPtr(1));
 
   delete Data;
 
 #if (LLVM_VERSION >= 40)
   if (&__archer_get_omp_status) {
static int ompt_tsan_initialize(ompt_function_lookup_t lookup,
   SET_CALLBACK_T(mutex_released, mutex);
   SET_OPTIONAL_CALLBACK_T(reduction, sync_region, hasReductionCallback, ompt_set_never);
 
   if (!tsan_flags.ignore_noninstrumented_modules)
     fprintf(
         stderr,
         "Warning: please export TSAN_OPTIONS='ignore_noninstrumented_modules=1' "
         "to avoid false positive reports from the OpenMP runtime!\n");
+  if (archer_flags->ignore_serial)
+    TsanIgnoreWritesBegin();
   return 1; // success
 }
 
 static void ompt_tsan_finalize(ompt_data_t *tool_data) {
+  if (archer_flags->ignore_serial)
+    TsanIgnoreWritesEnd();
   if (archer_flags->print_max_rss) {
     struct rusage end;
     getrusage(RUSAGE_SELF, &end);
     printf("MAX RSS[KBytes] during execution: %ld\n", end.ru_maxrss);
   }
 
   if (archer_flags)
     delete archer_flags;