[NFC][OpenMP][Offloading] Replaced explicit parallel level computation with function `__kmpc_parallel_level`

[tianshilei1992]

Summary

There are two places in current deviceRTLs where it computes parallel level explicitly,
which is basically the functionality of __kmpc_parallel_level. Starting from
D105787, we plan to introduce a series of function call folding based on information
that can be deducted during compilation time. Computation of parallel level is
the next target. This patch makes steps for the optimization.

[tianshilei1992]

openmp/libomptarget/deviceRTLs/common/src/libcall.cu

132

Because of the legacy from libomp, __kmpc_parallel_level takes two arguments: location and gtid, which are completely ignored in the function definition. As a result, we simply pass nullptr and 0.

[JonChesterfield]

kmpc_parallel_level is defined in parallel.cu as

EXTERN uint16_t __kmpc_parallel_level(kmp_Ident *loc, uint32_t global_tid) {
  PRINT0(LD_IO, "call to __kmpc_parallel_level\n");

  return parallelLevel[GetWarpId()] & (OMP_ACTIVE_PARALLEL_LEVEL - 1);
}

Similar to the last patch, we should drop the PRINT0 in that call before adding extra calls to it elsewhere.

The function interface is clang internal, let's change it to take void instead of a couple of ignored arguments. That will slightly simplify clang codegen and means the pass that looks for and replaces calls to it doesn't have to consider arguments (the latter may work out of the box)

[tianshilei1992]

In D105955#2876564, @JonChesterfield wrote:

The function interface is clang internal, let's change it to take void instead of a couple of ignored arguments. That will slightly simplify clang codegen and means the pass that looks for and replaces calls to it doesn't have to consider arguments (the latter may work out of the box)

That's probably not working because as far as I can tell, we share some code generation with host code, but for host code, the two arguments are not ignored.

[jdoerfert]

In D105955#2880542, @tianshilei1992 wrote:

In D105955#2876564, @JonChesterfield wrote:

The function interface is clang internal, let's change it to take void instead of a couple of ignored arguments. That will slightly simplify clang codegen and means the pass that looks for and replaces calls to it doesn't have to consider arguments (the latter may work out of the box)

That's probably not working because as far as I can tell, we share some code generation with host code, but for host code, the two arguments are not ignored.

I don't know what you mean by not ignored. the function is device only.

Drop the print and the arguments, also consider adding a new api for other parallel level lookups, we might even just reuse this one.

[tianshilei1992]

fixed comments

[tianshilei1992]

In D105955#2880868, @jdoerfert wrote:

In D105955#2880542, @tianshilei1992 wrote:

In D105955#2876564, @JonChesterfield wrote:

The function interface is clang internal, let's change it to take void instead of a couple of ignored arguments. That will slightly simplify clang codegen and means the pass that looks for and replaces calls to it doesn't have to consider arguments (the latter may work out of the box)

That's probably not working because as far as I can tell, we share some code generation with host code, but for host code, the two arguments are not ignored.

I don't know what you mean by not ignored. the function is device only.

Drop the print and the arguments, also consider adding a new api for other parallel level lookups, we might even just reuse this one.

Oh, I misunderstood the function here. I thought it just inherits from what we have in libomp as it keeps the same name style starting with __kmpc.

[jdoerfert]

LG, I'll add a patch to use the function more.

openmp/libomptarget/deviceRTLs/common/src/libcall.cu

 
 EXTERN int omp_get_max_active_levels(void) {
   int rc = 1;
   PRINT(LD_IO, "call omp_get_max_active_levels() returns %d\n", rc);
   return rc;
 }
 
 EXTERN int omp_get_level(void) {
-  int level = parallelLevel[GetWarpId()] & (OMP_ACTIVE_PARALLEL_LEVEL - 1);
+  int level = __kmpc_parallel_level();
   PRINT(LD_IO, "call omp_get_level() returns %d\n", level);
   return level;
 }
 
 EXTERN int omp_get_active_level(void) {
   int level = parallelLevel[GetWarpId()] > OMP_ACTIVE_PARALLEL_LEVEL ? 1 : 0;
   PRINT(LD_IO, "call omp_get_active_level() returns %d\n", level)
   return level;

openmp/libomptarget/deviceRTLs/common/src/parallel.cu

EXTERN void __kmpc_end_serialized_parallel(kmp_Ident *loc,
   omptarget_nvptx_threadPrivateContext->SetTopLevelTaskDescr(
       threadId, currTaskDescr->GetPrevTaskDescr());
   // free
   SafeFree(currTaskDescr, "new seq parallel task");
   currTaskDescr = getMyTopTaskDescriptor(threadId);
   currTaskDescr->RestoreLoopData();
 }
 
-EXTERN uint16_t __kmpc_parallel_level(kmp_Ident *loc, uint32_t global_tid) {
-  PRINT0(LD_IO, "call to __kmpc_parallel_level\n");
-
+EXTERN uint16_t __kmpc_parallel_level() {
   return parallelLevel[GetWarpId()] & (OMP_ACTIVE_PARALLEL_LEVEL - 1);
 }
 
 // This kmpc call returns the thread id across all teams. It's value is
 // cached by the compiler and used when calling the runtime. On nvptx
 // it's cheap to recalculate this value so we never use the result
 // of this call.
 EXTERN int32_t __kmpc_global_thread_num(kmp_Ident *loc) {
EXTERN void __kmpc_parallel_51(kmp_Ident *ident, kmp_int32 global_tid,
                                kmp_int32 if_expr, kmp_int32 num_threads,
                                int proc_bind, void *fn, void *wrapper_fn,
                                void **args, size_t nargs) {
 
   // Handle the serialized case first, same for SPMD/non-SPMD except that in
   // SPMD mode we already incremented the parallel level counter, account for
   // that.
   bool InParallelRegion =
-      (__kmpc_parallel_level(ident, global_tid) > __kmpc_is_spmd_exec_mode());
+      (__kmpc_parallel_level() > __kmpc_is_spmd_exec_mode());
   if (!if_expr || InParallelRegion) {
     __kmpc_serialized_parallel(ident, global_tid);
     __kmp_invoke_microtask(global_tid, 0, fn, args, nargs);
     __kmpc_end_serialized_parallel(ident, global_tid);
     return;
   }
 
   if (__kmpc_is_spmd_exec_mode()) {

openmp/libomptarget/deviceRTLs/common/src/support.cu

 ////////////////////////////////////////////////////////////////////////////////
 
 int GetOmpThreadId() {
   int tid = GetThreadIdInBlock();
   if (__kmpc_is_generic_main_thread(tid))
     return 0;
   // omp_thread_num
   int rc;
-  if ((parallelLevel[GetWarpId()] & (OMP_ACTIVE_PARALLEL_LEVEL - 1)) > 1) {
+  if (__kmpc_parallel_level() > 1) {
     rc = 0;
   } else if (__kmpc_is_spmd_exec_mode()) {
     rc = tid;
   } else {
     omptarget_nvptx_TaskDescr *currTaskDescr =
         omptarget_nvptx_threadPrivateContext->GetTopLevelTaskDescr(tid);
     ASSERT0(LT_FUSSY, currTaskDescr, "expected a top task descr");
     rc = currTaskDescr->ThreadId();

openmp/libomptarget/deviceRTLs/interface.h

 typedef enum omp_proc_bind_t {
   omp_proc_bind_false = 0,
   omp_proc_bind_true = 1,
   omp_proc_bind_master = 2,
   omp_proc_bind_close = 3,
   omp_proc_bind_spread = 4
 } omp_proc_bind_t;
 
 EXTERN double omp_get_wtick(void);
 EXTERN double omp_get_wtime(void);
 
 EXTERN void omp_set_num_threads(int num);
 EXTERN int omp_get_num_threads(void);
 EXTERN int omp_get_max_threads(void);
 EXTERN int omp_get_thread_limit(void);
 EXTERN int omp_get_thread_num(void);
 EXTERN int omp_get_num_procs(void);
 EXTERN int omp_in_parallel(void);
 EXTERN int omp_in_final(void);
 EXTERN void omp_set_dynamic(int flag);
 EXTERN int omp_get_dynamic(void);
 EXTERN void omp_set_nested(int flag);
 EXTERN int omp_get_nested(void);
 EXTERN void omp_set_max_active_levels(int level);
 EXTERN int omp_get_max_active_levels(void);
 EXTERN int omp_get_level(void);
 EXTERN int omp_get_active_level(void);
 EXTERN int omp_get_ancestor_thread_num(int level);
 EXTERN int omp_get_team_size(int level);
 
 EXTERN void omp_init_lock(omp_lock_t *lock);
 EXTERN void omp_init_nest_lock(omp_nest_lock_t *lock);
 EXTERN void omp_destroy_lock(omp_lock_t *lock);
 EXTERN void omp_destroy_nest_lock(omp_nest_lock_t *lock);
 EXTERN void omp_set_lock(omp_lock_t *lock);
 EXTERN void omp_set_nest_lock(omp_nest_lock_t *lock);

 ////////////////////////////////////////////////////////////////////////////////
 
 // parallel
 EXTERN int32_t __kmpc_global_thread_num(kmp_Ident *loc);
 EXTERN void __kmpc_push_num_threads(kmp_Ident *loc, int32_t global_tid,
                                     int32_t num_threads);
 EXTERN void __kmpc_serialized_parallel(kmp_Ident *loc, uint32_t global_tid);
 EXTERN void __kmpc_end_serialized_parallel(kmp_Ident *loc, uint32_t global_tid);
-EXTERN uint16_t __kmpc_parallel_level(kmp_Ident *loc, uint32_t global_tid);
+EXTERN uint16_t __kmpc_parallel_level();
 
 // proc bind
 EXTERN void __kmpc_push_proc_bind(kmp_Ident *loc, uint32_t global_tid,
                                   int proc_bind);
 EXTERN int omp_get_num_places(void);
 EXTERN int omp_get_place_num_procs(int place_num);
 EXTERN void omp_get_place_proc_ids(int place_num, int *ids);
 EXTERN int omp_get_place_num(void);