Set exit_frame in master task before calling a user's outlined function for a parallel loop construct

[jmellorcrummey]

Summary

For combined parallel loop constructs, the master task launches an outlined function directly from the runtime. In such cases, the OMPT frame object for the master task needs to have its exit_frame field set before the outlined function is called. This patch corrects an omission in the OMPT support

[protze.joachim]

Please provide test cases, so that we can understand the impact of this change.

runtime/src/kmp_gsupport.cpp

1094

inner_frame resolves to the same address as parent_frame in line 1087, so you could just use this variable.

The task executing the loop body is still the same task as the task executing the parallel region.

1096

This will overwrite the value written, when the parallel region begins.

runtime/src/kmp_gsupport.cpp

 #define OMPT_LOOP_PRE()                                                        \
   ompt_frame_t *parent_frame;                                                  \
   if (ompt_enabled.enabled) {                                                  \
     __ompt_get_task_info_internal(0, NULL, NULL, &parent_frame, NULL, NULL);   \
     parent_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);                 \
     OMPT_STORE_RETURN_ADDRESS(gtid);                                           \
   }
 
+#define OMPT_LOOP_BEFORE_TASK()                                                \
+  if (ompt_enabled.enabled) {                                                  \
+     ompt_frame_t *inner_frame;                                                \
+    __ompt_get_task_info_internal(0, NULL, NULL, &inner_frame, NULL, NULL);    \
+    inner_frame->exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);                   \
+  }
+
 #define OMPT_LOOP_POST()                                                       \
   if (ompt_enabled.enabled) {                                                  \
     parent_frame->enter_frame = ompt_data_none;                                \
   }
 
 #else
 
 #define OMPT_LOOP_PRE()
 
+#define OMPT_LOOP_BEFORE_TASK()
+
 #define OMPT_LOOP_POST()
 
 #endif
 
 PARALLEL_LOOP_START(
     KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_STATIC_START),
     kmp_sch_static, OMPT_LOOP_PRE, OMPT_LOOP_POST)
 PARALLEL_LOOP_START(
#endif
 
   KMP_DISPATCH_INIT(&loc, gtid, kmp_nm_dynamic_chunked, 1, count, 1, 1, TRUE);
 
   task(data);
   KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_END)();
   KA_TRACE(20, ("GOMP_parallel_sections exit: T#%d\n", gtid));
 }
 
-#define PARALLEL_LOOP(func, schedule, ompt_pre, ompt_post)                     \
+#define PARALLEL_LOOP(func, schedule, ompt_pre, ompt_before_task, ompt_post)   \
   void func(void (*task)(void *), void *data, unsigned num_threads, long lb,   \
             long ub, long str, long chunk_sz, unsigned flags) {                \
     int gtid = __kmp_entry_gtid();                                             \
     MKLOC(loc, KMP_STR(func));                                                 \
     KA_TRACE(                                                                  \
         20,                                                                    \
         (KMP_STR(                                                              \
              func) ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz 0x%lx\n",  \
void func(void (*task)(void *), void *data, unsigned num_threads, long lb,   \
     } else {                                                                   \
       __kmp_GOMP_serialized_parallel(&loc, gtid, task);                        \
     }                                                                          \
                                                                                \
     IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);)                          \
     KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb,                              \
                       (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz,          \
                       (schedule) != kmp_sch_static);                           \
+    ompt_before_task();                                                        \
     task(data);                                                                \
     KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_END)();                         \
     ompt_post();                                                               \
                                                                                \
     KA_TRACE(20, (KMP_STR(func) " exit: T#%d\n", gtid));                       \
   }
 
 PARALLEL_LOOP(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_STATIC),
-              kmp_sch_static, OMPT_LOOP_PRE, OMPT_LOOP_POST)
+              kmp_sch_static, OMPT_LOOP_PRE, OMPT_LOOP_BEFORE_TASK, 
+              OMPT_LOOP_POST)
 PARALLEL_LOOP(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_DYNAMIC),
-              kmp_sch_dynamic_chunked, OMPT_LOOP_PRE, OMPT_LOOP_POST)
+              kmp_sch_dynamic_chunked, OMPT_LOOP_PRE, OMPT_LOOP_BEFORE_TASK, 
+              OMPT_LOOP_POST)
 PARALLEL_LOOP(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_GUIDED),
-              kmp_sch_guided_chunked, OMPT_LOOP_PRE, OMPT_LOOP_POST)
+              kmp_sch_guided_chunked, OMPT_LOOP_PRE, OMPT_LOOP_BEFORE_TASK, 
+              OMPT_LOOP_POST)
 PARALLEL_LOOP(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_RUNTIME),
-              kmp_sch_runtime, OMPT_LOOP_PRE, OMPT_LOOP_POST)
+              kmp_sch_runtime, OMPT_LOOP_PRE, OMPT_LOOP_BEFORE_TASK, 
+              OMPT_LOOP_POST)
 
 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKGROUP_START)(void) {
   int gtid = __kmp_entry_gtid();
   MKLOC(loc, "GOMP_taskgroup_start");
   KA_TRACE(20, ("GOMP_taskgroup_start: T#%d\n", gtid));
 
 #if OMPT_SUPPORT
   if (ompt_enabled.enabled)