[OpenMP] [OMPD] openmp/runtime code changes required for OMPD

[Vigneshbalu]

Summary

This patch stems from “ompd-test” branch PRs https://github.com/OpenMPToolsInterface/llvm-project/commit/45e69d3a5fc27af7bf92ed5310987eb25070e538 and https://github.com/OpenMPToolsInterface/llvm-project/commit/e25e8f48b3573845f7a342ca02be3a5b3fac6090

These changes don't come under OMPD guard as it is a movement of existing code to capture parallel behavior correctly.

"Runtime Entry Points for OMPD" like "ompd_bp_parallel_begin" and "ompd_bp_parallel_begin" should be placed at the correct execution point for the debugging tool to access proper handles/data.

Without the below changes, in certain cases, debugging tool will pick the wrong parallel and task handle.

[Vigneshbalu]

rebasing the patch

[Vigneshbalu]

Rebasing the patch

[Vigneshbalu]

Gentle Reminder!

[Vigneshbalu]

Rebasing the patch.

[Vigneshbalu]

Updating the summary and the patch.

[Vigneshbalu]

Updating the diff full context.

[hbae]

LGTM

openmp/runtime/src/kmp_csupport.cpp

#endif
    * values */
   top = serial_team->t.t_control_stack_top;
   if (top && top->serial_nesting_level == serial_team->t.t_serialized) {
     copy_icvs(&serial_team->t.t_threads[0]->th.th_current_task->td_icvs, top);
     serial_team->t.t_control_stack_top = top->next;
     __kmp_free(top);
   }
 
-  // if( serial_team -> t.t_serialized > 1 )
-  serial_team->t.t_level--;
-
   /* pop dispatch buffers stack */
   KMP_DEBUG_ASSERT(serial_team->t.t_dispatch->th_disp_buffer);
   {
     dispatch_private_info_t *disp_buffer =
         serial_team->t.t_dispatch->th_disp_buffer;
     serial_team->t.t_dispatch->th_disp_buffer =
         serial_team->t.t_dispatch->th_disp_buffer->next;
     __kmp_free(disp_buffer);
   }
   this_thr->th.th_def_allocator = serial_team->t.t_def_allocator; // restore
 
   --serial_team->t.t_serialized;
   if (serial_team->t.t_serialized == 0) {
 
     /* return to the parallel section */
 
 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
     if (__kmp_inherit_fp_control && serial_team->t.t_fp_control_saved) {
       __kmp_clear_x87_fpu_status_word();
       __kmp_load_x87_fpu_control_word(&serial_team->t.t_x87_fpu_control_word);
       __kmp_load_mxcsr(&serial_team->t.t_mxcsr);
     }
 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
 
+    __kmp_pop_current_task_from_thread(this_thr);
 #if OMPD_SUPPORT
     if (ompd_state & OMPD_ENABLE_BP)
       ompd_bp_parallel_end();
 #endif
 
     this_thr->th.th_team = serial_team->t.t_parent;
     this_thr->th.th_info.ds.ds_tid = serial_team->t.t_master_tid;
 
     /* restore values cached in the thread */
     this_thr->th.th_team_nproc = serial_team->t.t_parent->t.t_nproc; /*  JPH */
     this_thr->th.th_team_master =
         serial_team->t.t_parent->t.t_threads[0]; /* JPH */
     this_thr->th.th_team_serialized = this_thr->th.th_team->t.t_serialized;
 
     /* TODO the below shouldn't need to be adjusted for serialized teams */
     this_thr->th.th_dispatch =
         &this_thr->th.th_team->t.t_dispatch[serial_team->t.t_master_tid];
 
-    __kmp_pop_current_task_from_thread(this_thr);
-
     KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 0);
     this_thr->th.th_current_task->td_flags.executing = 1;
 
     if (__kmp_tasking_mode != tskm_immediate_exec) {
       // Copy the task team from the new child / old parent team to the thread.
       this_thr->th.th_task_team =
           this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state];
       KA_TRACE(20,
                ("__kmpc_end_serialized_parallel: T#%d restoring task_team %p / "
                 "team %p\n",
                 global_tid, this_thr->th.th_task_team, this_thr->th.th_team));
     }
   } else {
     if (__kmp_tasking_mode != tskm_immediate_exec) {
       KA_TRACE(20, ("__kmpc_end_serialized_parallel: T#%d decreasing nesting "
                     "depth of serial team %p to %d\n",
                     global_tid, serial_team, serial_team->t.t_serialized));
     }
   }
 
+  serial_team->t.t_level--;
   if (__kmp_env_consistency_check)
     __kmp_pop_parallel(global_tid, NULL);
 #if OMPT_SUPPORT
   if (ompt_enabled.enabled)
     this_thr->th.ompt_thread_info.state =
         ((this_thr->th.th_team_serialized) ? ompt_state_work_serial
                                            : ompt_state_work_parallel);
 #endif

openmp/runtime/src/ompt-specific.cpp

if (always ||
     }
     link_lwt->heap = on_heap;
 
     // would be swap in the (on_stack) case.
     ompt_team_info_t tmp_team = lwt->ompt_team_info;
     link_lwt->ompt_team_info = *OMPT_CUR_TEAM_INFO(thr);
     *OMPT_CUR_TEAM_INFO(thr) = tmp_team;
 
-    ompt_task_info_t tmp_task = lwt->ompt_task_info;
-    link_lwt->ompt_task_info = *OMPT_CUR_TASK_INFO(thr);
-    *OMPT_CUR_TASK_INFO(thr) = tmp_task;
-
     // link the taskteam into the list of taskteams:
     ompt_lw_taskteam_t *my_parent =
         thr->th.th_team->t.ompt_serialized_team_info;
     link_lwt->parent = my_parent;
     thr->th.th_team->t.ompt_serialized_team_info = link_lwt;
 #if OMPD_SUPPORT
     if (ompd_state & OMPD_ENABLE_BP) {
       ompd_bp_parallel_begin();
     }
 #endif
+
+    ompt_task_info_t tmp_task = lwt->ompt_task_info;
+    link_lwt->ompt_task_info = *OMPT_CUR_TASK_INFO(thr);
+    *OMPT_CUR_TASK_INFO(thr) = tmp_task;
   } else {
     // this is the first serialized team, so we just store the values in the
     // team and drop the taskteam-object
     *OMPT_CUR_TEAM_INFO(thr) = lwt->ompt_team_info;
 #if OMPD_SUPPORT
     if (ompd_state & OMPD_ENABLE_BP) {
       ompd_bp_parallel_begin();
     }
 #endif
     *OMPT_CUR_TASK_INFO(thr) = lwt->ompt_task_info;
   }
 }
 
 void __ompt_lw_taskteam_unlink(kmp_info_t *thr) {
   ompt_lw_taskteam_t *lwtask = thr->th.th_team->t.ompt_serialized_team_info;
   if (lwtask) {
+    ompt_task_info_t tmp_task = lwtask->ompt_task_info;
+    lwtask->ompt_task_info = *OMPT_CUR_TASK_INFO(thr);
+    *OMPT_CUR_TASK_INFO(thr) = tmp_task;
 #if OMPD_SUPPORT
     if (ompd_state & OMPD_ENABLE_BP) {
       ompd_bp_parallel_end();
     }
 #endif
     thr->th.th_team->t.ompt_serialized_team_info = lwtask->parent;
 
     ompt_team_info_t tmp_team = lwtask->ompt_team_info;
     lwtask->ompt_team_info = *OMPT_CUR_TEAM_INFO(thr);
     *OMPT_CUR_TEAM_INFO(thr) = tmp_team;
 
-    ompt_task_info_t tmp_task = lwtask->ompt_task_info;
-    lwtask->ompt_task_info = *OMPT_CUR_TASK_INFO(thr);
-    *OMPT_CUR_TASK_INFO(thr) = tmp_task;
-
     if (lwtask->heap) {
       __kmp_free(lwtask);
       lwtask = NULL;
     }
   }
   //    return lwtask;
 }