[OpenMP][Tools] Teams support for libarcher

[protze.joachim]

Summary

Adding teams support for Archer:

• data management in initial-task-begin/end is different for the initial tasks in a league than for the initial task representing the OpenMP program.
• storing the flag from the parallel begin allows to distinguish league and parallel team
• according to OpenMP implicit flush semantics, the end of initial tasks in a league is before the program can continue execution at the end of the teams region.
• in contrast, for a parallel region, the synchronization happens in the implicit barrier and the parallel-end event should not be involved.

openmp/tools/archer/ompt-tsan.cpp

struct ParallelData final : DataPoolEntry<ParallelData> {
 
   // Parallel fork is just another barrier, use Barrier[1]
 
   /// Two addresses for relationships with barriers.
   ompt_tsan_clockid Barrier[2];
 
   const void *codePtr;
 
+  int Type{0};
+
+  bool isLeague() { return Type & ompt_parallel_league; }
+
   void *GetParallelPtr() { return &(Barrier[1]); }
 
   void *GetBarrierPtr(unsigned Index) { return &(Barrier[Index]); }
 
-  ParallelData *Init(const void *codeptr) {
+  ParallelData *Init(const void *codeptr, int type) {
     codePtr = codeptr;
+    Type = type;
     return this;
   }
 
   void Reset() {}
 
-  static ParallelData *New(const void *codeptr) {
-    return DataPoolEntry<ParallelData>::New()->Init(codeptr);
+  static ParallelData *New(const void *codeptr, int type) {
+    return DataPoolEntry<ParallelData>::New()->Init(codeptr, type);
   }
 
   ParallelData(DataPool<ParallelData> *dp) : DataPoolEntry<ParallelData>(dp) {}
 };
 
 static inline ParallelData *ToParallelData(ompt_data_t *parallel_data) {
   return reinterpret_cast<ParallelData *>(parallel_data->ptr);
 }
struct TaskData final : DataPoolEntry<TaskData> {
   bool InBarrier{false};
 
   /// Whether this task is an included task.
   int TaskType{0};
 
   /// count execution phase
   int execution{0};
 
+  /// Number of the thread in a team
+  int ThreadNum{0};
+
   /// Index of which barrier to use next.
   char BarrierIndex{0};
 
   /// Count how often this structure has been put into child tasks + 1.
   std::atomic_int RefCount{1};
 
   /// Reference to the parent that created this task.
   TaskData *Parent{nullptr};
if (Parent != nullptr) {
       Parent->RefCount++;
       // Copy over pointer to taskgroup. This task may set up its own stack
       // but for now belongs to its parent's taskgroup.
       TaskGroup = Parent->TaskGroup;
     }
     return this;
   }
 
-  TaskData *Init(ParallelData *team, int taskType) {
+  TaskData *Init(ParallelData *team, int taskType, int threadNum) {
     TaskType = taskType;
     execution = 1;
+    ThreadNum = threadNum;
     ImplicitTask = this;
     Team = team;
     return this;
   }
 
   void Reset() {
     InBarrier = false;
     TaskType = 0;
     execution = 0;
+    ThreadNum = 0;
     BarrierIndex = 0;
     RefCount = 1;
     Parent = nullptr;
     ImplicitTask = nullptr;
     Team = nullptr;
     TaskGroup = nullptr;
     if (DependencyMap) {
       for (auto i : *DependencyMap)
#ifdef DEBUG
     freed = 0;
 #endif
   }
 
   static TaskData *New(TaskData *parent, int taskType) {
     return DataPoolEntry<TaskData>::New()->Init(parent, taskType);
   }
 
-  static TaskData *New(ParallelData *team, int taskType) {
-    return DataPoolEntry<TaskData>::New()->Init(team, taskType);
+  static TaskData *New(ParallelData *team, int taskType, int threadNum) {
+    return DataPoolEntry<TaskData>::New()->Init(team, taskType, threadNum);
   }
 
   TaskData(DataPool<TaskData> *dp) : DataPoolEntry<TaskData>(dp) {}
 };
 
 static inline TaskData *ToTaskData(ompt_data_t *task_data) {
   return reinterpret_cast<TaskData *>(task_data->ptr);
 }

 
 /// OMPT event callbacks for handling parallel regions.
 
 static void ompt_tsan_parallel_begin(ompt_data_t *parent_task_data,
                                      const ompt_frame_t *parent_task_frame,
                                      ompt_data_t *parallel_data,
                                      uint32_t requested_team_size, int flag,
                                      const void *codeptr_ra) {
-  ParallelData *Data = ParallelData::New(codeptr_ra);
+  ParallelData *Data = ParallelData::New(codeptr_ra, flag);
   parallel_data->ptr = Data;
 
   TsanHappensBefore(Data->GetParallelPtr());
   if (archer_flags->ignore_serial && ToTaskData(parent_task_data)->isInitial())
     TsanIgnoreWritesEnd();
 }
 
 static void ompt_tsan_parallel_end(ompt_data_t *parallel_data,
                                    ompt_data_t *task_data, int flag,
                                    const void *codeptr_ra) {
   if (archer_flags->ignore_serial && ToTaskData(task_data)->isInitial())
     TsanIgnoreWritesBegin();
   ParallelData *Data = ToParallelData(parallel_data);
-  TsanHappensAfter(Data->GetBarrierPtr(0));
-  TsanHappensAfter(Data->GetBarrierPtr(1));
+  if (Data->isLeague())
+    TsanHappensAfter(Data->GetParallelPtr());
 
   Data->Delete();
 
 #if (LLVM_VERSION >= 40)
   if (&__archer_get_omp_status) {
     if (__archer_get_omp_status() == 0 && archer_flags->flush_shadow)
       __tsan_flush_memory();
   }
 #endif
 }
 
 static void ompt_tsan_implicit_task(ompt_scope_endpoint_t endpoint,
                                     ompt_data_t *parallel_data,
                                     ompt_data_t *task_data,
                                     unsigned int team_size,
                                     unsigned int thread_num, int type) {
   switch (endpoint) {
   case ompt_scope_begin:
-    if (type & ompt_task_initial) {
-      parallel_data->ptr = ParallelData::New(nullptr);
+    if (type & ompt_task_initial && parallel_data->ptr == NULL) {
+      parallel_data->ptr = ParallelData::New(nullptr, ompt_parallel_team);
     }
-    task_data->ptr = TaskData::New(ToParallelData(parallel_data), type);
+    task_data->ptr =
+        TaskData::New(ToParallelData(parallel_data), type, thread_num);
     TsanHappensAfter(ToParallelData(parallel_data)->GetParallelPtr());
     TsanFuncEntry(ToParallelData(parallel_data)->codePtr);
     break;
   case ompt_scope_end: {
     TaskData *Data = ToTaskData(task_data);
 #ifdef DEBUG
     assert(Data->freed == 0 && "Implicit task end should only be called once!");
     Data->freed = 1;
 #endif
     assert(Data->RefCount == 1 &&
            "All tasks should have finished at the implicit barrier!");
     if (type & ompt_task_initial) {
+      if (Data->Team->isLeague()) {
+        TsanHappensBefore(Data->Team->GetParallelPtr());
+      } else {
         Data->Team->Delete();
       }
+    }
     Data->Delete();
     TsanFuncExit();
     break;
   }
   case ompt_scope_beginend:
     // Should not occur according to OpenMP 5.1
     // Tested in OMPT tests
     break;
case ompt_sync_region_barrier: {
       if (hasReductionCallback < ompt_set_always) {
         // We want to track writes after the barrier again.
         Data->InBarrier = false;
         TsanIgnoreWritesEnd();
       }
 
       char BarrierIndex = Data->BarrierIndex;
       // Barrier will end after it has been entered by all threads.
-      if (parallel_data)
+      if (parallel_data || Data->ThreadNum == 0)
         TsanHappensAfter(Data->Team->GetBarrierPtr(BarrierIndex));
 
       // It is not guaranteed that all threads have exited this barrier before
       // we enter the next one. So we will use a different address.
       // We are however guaranteed that this current barrier is finished
       // by the time we exit the next one. So we can then reuse the first
       // address.
       Data->BarrierIndex = (BarrierIndex + 1) % 2;
static void ompt_tsan_task_create(
     const ompt_frame_t *parent_frame, /* frame data for parent task   */
     ompt_data_t *new_task_data,       /* id of created task           */
     int type, int has_dependences,
     const void *codeptr_ra) /* pointer to outlined function */
 {
   TaskData *Data;
   assert(new_task_data->ptr == NULL &&
          "Task data should be initialized to NULL");
+  // This case tries to maintain some pre-5.0 backwards compatibility
   if (type & ompt_task_initial) {
     ompt_data_t *parallel_data;
     int team_size = 1;
     ompt_get_parallel_info(0, &parallel_data, &team_size);
-    ParallelData *PData = ParallelData::New(nullptr);
+    ParallelData *PData = ParallelData::New(nullptr, ompt_parallel_team);
     parallel_data->ptr = PData;
 
-    Data = TaskData::New(PData, type);
+    Data = TaskData::New(PData, type, 0);
     new_task_data->ptr = Data;
   } else if (type & ompt_task_undeferred) {
     Data = TaskData::New(ToTaskData(parent_task_data), type);
     new_task_data->ptr = Data;
   } else if (type & ompt_task_explicit || type & ompt_task_target) {
     Data = TaskData::New(ToTaskData(parent_task_data), type);
     new_task_data->ptr = Data;