[OMPT] Fix test tasks/serialized.c with optimization

[Hahnfeld]

Summary

The compiler inlines the user code in the task. Check for that case at
runtime by comparing the frame addresses and print the expected exit
address.

Also showcase how I think the OMPT tests could be reformatted to match
LLVM's code style. In my opinion it would be great to that kind of change
to all tests that need to be touched for whatever reason...

[omalyshe]

Found a small issue with CHECK.
All other LGTM.

runtime/test/ompt/tasks/serialized.c

146

Looks like parallel_id should be checked for PARALLEL_ID here.
Same on line 149.

[Hahnfeld]

runtime/test/ompt/tasks/serialized.c

146

This doesn't work, parallel_id=0 for these two events. I'd guess that's because the parallel region ended in the barrier, @protze.joachim?

[omalyshe]

This doesn't work, parallel_id=0 for these two events

Indeed. parallel_end event somehow goes before barrier_end and imlicit_task_end
...
281474976710657: ompt_event_barrier_end: parallel_id=281474976710660, task_id=281474976710661, codeptr_ra=0x40131b
281474976710657: ompt_event_implicit_task_end: parallel_id=0, task_id=281474976710661, team_size=2, thread_num=0
281474976710657: ompt_event_parallel_end: parallel_id=281474976710660, task_id=281474976710658, invoker=2, codeptr_ra=0x40131b
281474976710657: ompt_event_thread_end: thread_id=281474976710657
562949953421313: ompt_event_wait_barrier_end: parallel_id=0, task_id=562949953421314, codeptr_ra=(nil)
562949953421313: ompt_event_barrier_end: parallel_id=0, task_id=562949953421314, codeptr_ra=(nil)
562949953421313: ompt_event_implicit_task_end: parallel_id=0, task_id=562949953421314, team_size=0, thread_num=1
...

[protze.joachim]

This behavior is right (p.417,l.5):

The binding of argument parallel_data is the current parallel region. For the implicit-task-end
event, this argument is NULL.

And since we cannot access the ID, if NULL is passed as the address of parallel_data, the callback prints 0.

The reason for this behavior is that barrier-end and implicit-task-end of worker threads is not synchronized with the parallel-end of the master and therefore the parallel-data is most probably gone at that point.

runtime/test/ompt/tasks/serialized.c

146

Yes, parallel_id=0 would be the right check for all worker threads.

146

Sorry, for all threads.

[Hahnfeld]

Test parallel_id in {implicit_task,barrier}_end events. Note that this reveals a bug that I intend to fix separately.

runtime/test/ompt/tasks/serialized.c

146

Ok, that turns out to be a bug for master threads. I've posted a fix in D43308.

[Hahnfeld]

Ping, anything else left?

[protze.joachim]

LGTM

runtime/test/ompt/callback.h

static void print_ids(int level)
   if (frame)
   {
     printf("%" PRIu64 ": task level %d: parallel_id=%" PRIu64 ", task_id=%" PRIu64 ", exit_frame=%p, reenter_frame=%p\n", ompt_get_thread_data()->value, level, exists_task ? parallel_data->value : 0, exists_task ? task_data->value : 0, frame->exit_frame, frame->enter_frame);
   }
   else
     printf("%" PRIu64 ": task level %d: parallel_id=%" PRIu64 ", task_id=%" PRIu64 ", frame=%p\n", ompt_get_thread_data()->value, level, exists_task ? parallel_data->value : 0, exists_task ? task_data->value : 0, frame);
 }
 
-#define print_frame(level)\
-do {\
-  printf("%" PRIu64 ": __builtin_frame_address(%d)=%p\n", ompt_get_thread_data()->value, level, __builtin_frame_address(level));\
-} while(0)
+#define get_frame_address(level) __builtin_frame_address(level)
+
+#define print_frame(level)                                                     \
+  printf("%" PRIu64 ": __builtin_frame_address(%d)=%p\n",                      \
+         ompt_get_thread_data()->value, level, get_frame_address(level))
 
 // clang (version 5.0 and above) adds an intermediate function call with debug flag (-g)
 #if defined(TEST_NEED_PRINT_FRAME_FROM_OUTLINED_FN)
   #if defined(DEBUG) && defined(__clang__) && __clang_major__ >= 5
     #define print_frame_from_outlined_fn(level) print_frame(level+1)
   #else
     #define print_frame_from_outlined_fn(level) print_frame(level)
   #endif

runtime/test/ompt/tasks/serialized.c

 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
 // UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7
 #define TEST_NEED_PRINT_FRAME_FROM_OUTLINED_FN
 #include "callback.h"
-#include <omp.h>   
+#include <omp.h>
 #include <math.h>
 
-int main()
-{
+int main() {
   omp_set_nested(0);
   print_frame(0);
-  #pragma omp parallel num_threads(2)
   {
     print_frame_from_outlined_fn(1);
     print_ids(0);
     print_ids(1);
     print_frame(0);
-    #pragma omp master
     {
       print_ids(0);
+      void *creator_frame = get_frame_address(0);
       int t = (int)sin(0.1);
-      #pragma omp task if(t)
+#pragma omp task if (t)
       {
-        print_frame(1);
+        void *task_frame = get_frame_address(0);
+        if (creator_frame == task_frame) {
+          // Assume this code was inlined which the compiler is allowed to do.
+          print_frame(0);
+        } else {
+          // The exit frame must be our parent!
+          print_frame_from_outlined_fn(1);
+        }
         print_ids(0);
         print_ids(1);
         print_ids(2);
       }
       print_fuzzy_address(1);
       print_ids(0);
     }
     print_ids(0);
   }
 
-
   // Check if libomp supports the callbacks for this test.
-  // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_task_create'
-  // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_task_schedule'
-  // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_parallel_begin'
-  // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_parallel_end'
-  // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_implicit_task'
-  // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_mutex_acquire'
-  // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_mutex_acquired'
-  // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_mutex_released'
-
-  
+  // CHECK-NOT: {{^}}0: Could not register callback
   // CHECK: {{^}}0: NULL_POINTER=[[NULL:.*$]]
 
   // make sure initial data pointers are null
   // CHECK-NOT: 0: new_task_data initially not null
-  
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_task_create: parent_task_id={{[0-9]+}}, parent_task_frame.exit=[[NULL]], parent_task_frame.reenter=[[NULL]], new_task_id={{[0-9]+}}, codeptr_ra=[[NULL]], task_type=ompt_task_initial=1, has_dependences=no
-  // CHECK: {{^}}[[MASTER_ID]]: __builtin_frame_address(0)=[[MAIN_REENTER:0x[0-f]+]]
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-9]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter=[[MAIN_REENTER]], parallel_id=[[PARALLEL_ID:[0-9]+]], requested_team_size=2, codeptr_ra=0x{{[0-f]+}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_task_create
+  // CHECK-SAME: parent_task_id={{[0-9]+}}, parent_task_frame.exit=[[NULL]]
+  // CHECK-SAME: parent_task_frame.reenter=[[NULL]]
+  // CHECK-SAME: new_task_id={{[0-9]+}}, codeptr_ra=[[NULL]]
+  // CHECK-SAME: task_type=ompt_task_initial=1, has_dependences=no
+  // CHECK: {{^}}[[MASTER_ID]]: __builtin_frame_address(0)
+  // CHECK-SAME: =[[MAIN_REENTER:0x[0-f]+]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin
+  // CHECK-SAME: parent_task_id=[[PARENT_TASK_ID:[0-9]+]]
+  // CHECK-SAME: parent_task_frame.exit=[[NULL]]
+  // CHECK-SAME: parent_task_frame.reenter=[[MAIN_REENTER]]
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-9]+]], requested_team_size=2
+  // CHECK-SAME: codeptr_ra=0x{{[0-f]+}}, invoker={{[0-9]+}}
 
   // nested parallel masters
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
-  // CHECK: {{^}}[[MASTER_ID]]: __builtin_frame_address({{.}})=[[EXIT:0x[0-f]+]]
-  // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT]], reenter_frame=[[NULL]]
-  // CHECK: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-9]+]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[MAIN_REENTER]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]]
+  // CHECK-SAME: task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK: {{^}}[[MASTER_ID]]: __builtin_frame_address
+  // CHECK-SAME: =[[EXIT:0x[0-f]+]]
+
+  // CHECK: {{^}}[[MASTER_ID]]: task level 0
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK-SAME: exit_frame=[[EXIT]], reenter_frame=[[NULL]]
+
+  // CHECK: {{^}}[[MASTER_ID]]: task level 1
+  // CHECK-SAME: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-9]+]]
+  // CHECK-SAME: task_id=[[PARENT_TASK_ID]],
+  // CHECK-SAME: exit_frame=[[NULL]], reenter_frame=[[MAIN_REENTER]]
+
   // CHECK: {{^}}[[MASTER_ID]]: __builtin_frame_address(0)=[[REENTER:0x[0-f]+]]
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit=[[EXIT]], parent_task_frame.reenter=[[REENTER]], new_task_id=[[TASK_ID:[0-9]+]], codeptr_ra=[[RETURN_ADDRESS:0x[0-f]+]]{{[0-f][0-f]}}
-  // <- ompt_event_task_schedule ([[IMPLICIT_TASK_ID]], [[TASK_ID]]) would be expected here
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_schedule: first_task_id=[[IMPLICIT_TASK_ID]], second_task_id=[[TASK_ID]]
-  // CHECK: {{^}}[[MASTER_ID]]: __builtin_frame_address(1)=[[TASK_EXIT:0x[0-f]+]]
-  // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID]], exit_frame=[[TASK_EXIT]], reenter_frame=[[NULL]]
-  // CHECK: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT]], reenter_frame=[[REENTER]]
-  // CHECK: {{^}}[[MASTER_ID]]: task level 2: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[MAIN_REENTER]]
-  // <- ompt_event_task_schedule ([[TASK_ID]], [[IMPLICIT_TASK_ID]]) would be expected here
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_schedule: first_task_id=[[TASK_ID]], second_task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create
+  // CHECK-SAME: parent_task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK-SAME: parent_task_frame.exit=[[EXIT]]
+  // CHECK-SAME: parent_task_frame.reenter=[[REENTER]]
+  // CHECK-SAME: new_task_id=[[TASK_ID:[0-9]+]]
+  // CHECK-SAME: codeptr_ra=[[RETURN_ADDRESS:0x[0-f]+]]{{[0-f][0-f]}}
+
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_schedule:
+  // CHECK-SAME: first_task_id=[[IMPLICIT_TASK_ID]], second_task_id=[[TASK_ID]]
+  // CHECK: {{^}}[[MASTER_ID]]: __builtin_frame_address
+  // CHECK-SAME: =[[TASK_EXIT:0x[0-f]+]]
+  // CHECK: {{^}}[[MASTER_ID]]: task level 0
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID]]
+  // CHECK-SAME: exit_frame=[[TASK_EXIT]], reenter_frame=[[NULL]]
+
+  // CHECK: {{^}}[[MASTER_ID]]: task level 1
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK-SAME: exit_frame=[[EXIT]], reenter_frame=[[REENTER]]
+
+  // CHECK: {{^}}[[MASTER_ID]]: task level 2
+  // CHECK-SAME: parallel_id=[[IMPLICIT_PARALLEL_ID]]
+  // CHECK-SAME: task_id=[[PARENT_TASK_ID]]
+  // CHECK-SAME: exit_frame=[[NULL]], reenter_frame=[[MAIN_REENTER]]
+
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_schedule
+  // CHECK-SAME: first_task_id=[[TASK_ID]], second_task_id=[[IMPLICIT_TASK_ID]]
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_end: task_id=[[TASK_ID]]
   // CHECK: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
-  // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT]], reen
 
-  // implicit barrier parallel
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[NULL]]
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-
-  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
-  // CHECK: {{^}}[[THREAD_ID]]: __builtin_frame_address({{.}})=[[EXIT:0x[0-f]+]]
-  // CHECK: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT]], reenter_frame=[[NULL]]
-  // CHECK: {{^}}[[THREAD_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[MAIN_REENTER]]
-  // CHECK: {{^}}[[THREAD_ID]]: __builtin_frame_address(0)=[[REENTER:0x[0-f]+]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // CHECK: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[NULL]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK: {{^}}[[MASTER_ID]]: task level 0
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK-SAME: exit_frame=[[EXIT]], reenter_frame=[[NULL]]
 
+  // implicit barrier parallel
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_begin
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK: {{^}}[[MASTER_ID]]: task level 0
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK-SAME: exit_frame=[[NULL]], reenter_frame=[[NULL]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_end
+  // CHECK-SAME: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end
+  // CHECK-SAME: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+
+  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]]
+  // CHECK-SAME: task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK: {{^}}[[THREAD_ID]]: __builtin_frame_address
+  // CHECK-SAME: =[[EXIT:0x[0-f]+]]
+  // CHECK: {{^}}[[THREAD_ID]]: task level 0
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK-SAME: exit_frame=[[EXIT]], reenter_frame=[[NULL]]
+  // CHECK: {{^}}[[THREAD_ID]]: task level 1
+  // CHECK-SAME: parallel_id=[[IMPLICIT_PARALLEL_ID]]
+  // CHECK-SAME: task_id=[[PARENT_TASK_ID]]
+  // CHECK-SAME: exit_frame=[[NULL]], reenter_frame=[[MAIN_REENTER]]
+
+  // CHECK: {{^}}[[THREAD_ID]]: __builtin_frame_address(0)={{0x[0-f]+}}
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_begin
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK: {{^}}[[THREAD_ID]]: task level 0
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK-SAME: exit_frame=[[NULL]], reenter_frame=[[NULL]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_end
+  // CHECK-SAME: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
+  // CHECK-SAME: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
   return 0;
 }