[OpenMP][1/3] Introduce the KernelEnvironment into the device runtimes

[jdoerfert]

Summary

This introduces the kernel environment which contains information passed
by the compiler to a GPU kernel. For now it mostly encapsulated the
ident_t object and the execution configuration, thus information we
passed explicitly before. We will add more content, including mutable
content similar to the debug indention level, later on.

[jdoerfert]

openmp/libomptarget/DeviceRTL/src/Debug.cpp

76

[jhuber6]

LGTM

[ye-luo]

It sounds like such info will be passed from host to the device once per kernel and the performance impact is negligible. Right?

[jdoerfert]

In D116908#3262836, @ye-luo wrote:

It sounds like such info will be passed from host to the device once per kernel and the performance impact is negligible. Right?

The information is directly baked into the device image (in form of globals).
The transfer happens as you load the image, basically a few extra bytes per kernel but we had most of the things before as separate globals already anyway.
There is no kernel start cost to speak of, at least I don't expect any.

[ye-luo]

In D116908#3262918, @jdoerfert wrote:

In D116908#3262836, @ye-luo wrote:

It sounds like such info will be passed from host to the device once per kernel and the performance impact is negligible. Right?

The information is directly baked into the device image (in form of globals).
The transfer happens as you load the image, basically a few extra bytes per kernel but we had most of the things before as separate globals already anyway.
There is no kernel start cost to speak of, at least I don't expect any.

Thanks for the clarification.

[jdoerfert]

Subsumed by D142569

llvm/include/llvm/Frontend/OpenMP/ConfigurationEnvironment.h

+//===- ConfigurationEnvironment.h - OpenMP GPU config environment - C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OMPTARGET_CONFIGURATION_ENVIRONMENT_H
+#define OMPTARGET_CONFIGURATION_ENVIRONMENT_H
+
+// deviceRTL uses <stdint> and DeviceRTL uses explicit definitions
+
+#include "Environment.h"
+#include "OMPConstants.h"
+
+#ifdef OMPTARGET_DEVICE_RUNTIME
+namespace _OMP {
+#endif
+
+struct ConfigurationEnvironmentTy {
+  uint8_t UseGenericStateMachine;
+
+  llvm::omp::OMPTgtExecModeFlags ExecMode;
+};
+
+#ifdef OMPTARGET_DEVICE_RUNTIME
+} // namespace _OMP
+#endif
+
+#endif

llvm/include/llvm/Frontend/OpenMP/Environment.h

+//===- Environment.h - OpenMP GPU environment helper declarations - C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OMPTARGET_ENVIRONMENT_H
+#define OMPTARGET_ENVIRONMENT_H
+
+// deviceRTL uses <stdint> and DeviceRTL uses explicit definitions
+
+#ifdef OMPTARGET_DEVICE_RUNTIME
+#include "Types.h"
+#else
+#ifdef OMPTARGET_OLD_DEVICE_RUNTIME
+#include "interface.h"
+using IdentTy = ident_t;
+#else
+#include "SourceInfo.h"
+using IdentTy = ident_t;
+#endif
+#endif
+
+#endif

llvm/include/llvm/Frontend/OpenMP/KernelEnvironment.h

+//===---- KernelEnvironment.h - OpenMP GPU kernel environment ----- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OMPTARGET_KERNEL_ENVIRONMENT_H
+#define OMPTARGET_KERNEL_ENVIRONMENT_H
+
+// deviceRTL uses <stdint> and DeviceRTL uses explicit definitions
+
+#include "ConfigurationEnvironment.h"
+#include "Environment.h"
+
+#ifdef OMPTARGET_DEVICE_RUNTIME
+namespace _OMP {
+#endif
+
+struct KernelEnvironmentTy {
+  IdentTy Ident;
+
+  ConfigurationEnvironmentTy Configuration;
+
+  /// Current indentation level for the function trace. Only accessed by thread
+  /// 0.
+  uint16_t DebugIndentionLevel;
+};
+
+#ifdef OMPTARGET_DEVICE_RUNTIME
+} // namespace _OMP
+#endif
+
+#endif

openmp/libomptarget/DeviceRTL/include/Interface.h

 //===-------- Interface.h - OpenMP interface ---------------------- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef OMPTARGET_DEVICERTL_INTERFACE_H
 #define OMPTARGET_DEVICERTL_INTERFACE_H
 
 #include "Types.h"
+#include "llvm/Frontend/OpenMP/KernelEnvironment.h"
 
 /// External API
 ///
 ///{
 
 extern "C" {
 
 /// ICV: dyn-var, constant 0

 /// External interface to get the warp size.
 uint32_t __kmpc_get_warp_size();
 
 /// Kernel
 ///
 ///{
 int8_t __kmpc_is_spmd_exec_mode();
 
-int32_t __kmpc_target_init(IdentTy *Ident, int8_t Mode,
-                           bool UseGenericStateMachine, bool);
+int32_t __kmpc_target_init(_OMP::KernelEnvironmentTy &KernelEnv, bool);
 
-void __kmpc_target_deinit(IdentTy *Ident, int8_t Mode, bool);
+void __kmpc_target_deinit(bool);
 
 ///}
 
 /// Reduction
 ///
 ///{
 void __kmpc_nvptx_end_reduce(int32_t TId);
 

openmp/libomptarget/DeviceRTL/include/State.h

 //===-------- State.h - OpenMP State & ICV interface ------------- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef OMPTARGET_STATE_H
 #define OMPTARGET_STATE_H
 
 #include "Debug.h"
 #include "Types.h"
+#include "llvm/Frontend/OpenMP/KernelEnvironment.h"
 
 #pragma omp declare target
 
 namespace _OMP {
 
 namespace state {
 
 inline constexpr uint32_t SharedScratchpadSize = SHARED_SCRATCHPAD_SIZE;
 
 /// Initialize the state machinery. Must be called by all threads.
-void init(bool IsSPMD);
+void init(bool IsSPMD, KernelEnvironmentTy &KernelEnv);
+
+/// Return the kernel environment associated with the current kernel.
+KernelEnvironmentTy &getKernelEnvironment();
 
 /// TODO
 enum ValueKind {
   VK_NThreads,
   VK_Level,
   VK_ActiveLevel,
   VK_MaxActiveLevels,
   VK_RunSched,

openmp/libomptarget/DeviceRTL/src/Debug.cpp

 //===--- Debug.cpp -------- Debug utilities ----------------------- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file contains debug utilities
 //
 //===----------------------------------------------------------------------===//
 
 #include "Debug.h"
 #include "Configuration.h"
 #include "Interface.h"
 #include "Mapping.h"
+#include "State.h"
 #include "Types.h"
 
 using namespace _OMP;
 
 #pragma omp declare target
 
 extern "C" {
 void __assert_assume(bool condition) { __builtin_assume(condition); }

 } // namespace impl
 #pragma omp end declare variant
 
 int32_t __llvm_omp_vprintf(const char *Format, void *Arguments, uint32_t Size) {
   return impl::omp_vprintf(Format, Arguments, Size);
 }
 }
 
-/// Current indentation level for the function trace. Only accessed by thread 0.
-__attribute__((loader_uninitialized))
-static uint32_t Level;
-#pragma omp allocate(Level) allocator(omp_pteam_mem_alloc)
-
 DebugEntryRAII::DebugEntryRAII(const char *File, const unsigned Line,
                                const char *Function) {
   if (config::isDebugMode(config::DebugKind::FunctionTracing) &&
       mapping::getThreadIdInBlock() == 0 && mapping::getBlockId() == 0) {
+    uint16_t &Level = state::getKernelEnvironment().DebugIndentionLevel;
 
     for (int I = 0; I < Level; ++I)
       PRINTF("%s", "  ");
 
     PRINTF("%s:%u: Thread %u Entering %s\n", File, Line,
            mapping::getThreadIdInBlock(), Function);
     Level++;
   }
 }
 
 DebugEntryRAII::~DebugEntryRAII() {
   if (config::isDebugMode(config::DebugKind::FunctionTracing) &&
-      mapping::getThreadIdInBlock() == 0 && mapping::getBlockId() == 0)
+      mapping::getThreadIdInBlock() == 0 && mapping::getBlockId() == 0) {
+    uint16_t Level = state::getKernelEnvironment().DebugIndentionLevel;
     Level--;
   }
+}
 
 void DebugEntryRAII::init() { Level = 0; }
 
 #pragma omp end declare target

openmp/libomptarget/DeviceRTL/src/Kernel.cpp

 //===----------------------------------------------------------------------===//
 
 #include "Debug.h"
 #include "Interface.h"
 #include "Mapping.h"
 #include "State.h"
 #include "Synchronization.h"
 #include "Types.h"
+#include "llvm/Frontend/OpenMP/KernelEnvironment.h"
 
 using namespace _OMP;
 
 #pragma omp declare target
 
-static void inititializeRuntime(bool IsSPMD) {
+static void inititializeRuntime(bool IsSPMD, KernelEnvironmentTy &KernelEnv) {
   // Order is important here.
   synchronize::init(IsSPMD);
   mapping::init(IsSPMD);
-  state::init(IsSPMD);
+  state::init(IsSPMD, KernelEnv);
 }
 
 /// Simple generic state machine for worker threads.
-static void genericStateMachine(IdentTy *Ident) {
+static void genericStateMachine(IdentTy &Ident) {
   FunctionTracingRAII();
 
   uint32_t TId = mapping::getThreadIdInBlock();
 
   do {
     ParallelRegionFnTy WorkFn = 0;
 
     // Wait for the signal that we have a new work function.
do {
 
     synchronize::threads();
 
   } while (true);
 }
 
 extern "C" {
 
-/// Initialization
-///
-/// \param Ident               Source location identification, can be NULL.
-///
-int32_t __kmpc_target_init(IdentTy *Ident, int8_t Mode,
-                           bool UseGenericStateMachine, bool) {
+int32_t __kmpc_target_init(KernelEnvironmentTy &KernelEnv, bool) {
   FunctionTracingRAII();
-  const bool IsSPMD = Mode & OMP_TGT_EXEC_MODE_SPMD;
+  bool IsSPMD = KernelEnv.Configuration.ExecMode & OMP_TGT_EXEC_MODE_SPMD;
+  bool UseGenericStateMachine = KernelEnv.Configuration.UseGenericStateMachine;
   if (IsSPMD) {
-    inititializeRuntime(/* IsSPMD */ true);
+    inititializeRuntime(/* IsSPMD */ true, KernelEnv);
     synchronize::threadsAligned();
   } else {
-    inititializeRuntime(/* IsSPMD */ false);
+    inititializeRuntime(/* IsSPMD */ false, KernelEnv);
     // No need to wait since only the main threads will execute user
     // code and workers will run into a barrier right away.
   }
 
   if (IsSPMD) {
     state::assumeInitialState(IsSPMD);
     return -1;
   }
int32_t __kmpc_target_init(KernelEnvironmentTy &KernelEnv, bool) {
   // synchronize::threads call in genericStateMachine before the main thread
   // reaches its corresponding synchronize::threads call: that would permit all
   // active worker threads to proceed before the main thread has actually set
   // state::ParallelRegionFn, and then they would immediately quit without
   // doing any work.  mapping::getBlockSize() does not include any of the main
   // thread's warp, so none of its threads can ever be active worker threads.
   if (UseGenericStateMachine &&
       mapping::getThreadIdInBlock() < mapping::getBlockSize())
-    genericStateMachine(Ident);
+    genericStateMachine(KernelEnv.Ident);
 
   return mapping::getThreadIdInBlock();
 }
 
 /// De-Initialization
 ///
 /// In non-SPMD, this function releases the workers trapped in a state machine
 /// and also any memory dynamically allocated by the runtime.
-///
-/// \param Ident Source location identification, can be NULL.
-///
-void __kmpc_target_deinit(IdentTy *Ident, int8_t Mode, bool) {
+void __kmpc_target_deinit(bool) {
   FunctionTracingRAII();
-  const bool IsSPMD = Mode & OMP_TGT_EXEC_MODE_SPMD;
+  const bool IsSPMD = mapping::isSPMDMode();
   state::assumeInitialState(IsSPMD);
   if (IsSPMD)
     return;
 
   // Signal the workers to exit the state machine and exit the kernel.
   state::ParallelRegionFn = nullptr;
 }
 
 int8_t __kmpc_is_spmd_exec_mode() {
   FunctionTracingRAII();
   return mapping::isSPMDMode();
 }
 }
 
 #pragma omp end declare target

openmp/libomptarget/DeviceRTL/src/State.cpp

 /// FIXME: The stack shouldn't require worst-case padding. Alignment needs to be
 /// passed in as an argument and the stack rewritten to support it.
 constexpr const uint32_t Alignment = 16;
 
 /// External symbol to access dynamic shared memory.
 extern unsigned char DynamicSharedBuffer[] __attribute__((aligned(Alignment)));
 #pragma omp allocate(DynamicSharedBuffer) allocator(omp_pteam_mem_alloc)
 
+/// The kernel environment passed to the init method by the compiler.
+static KernelEnvironmentTy *SHARED(KernelEnvironmentPtr);
+
 namespace {
 
 /// Fallback implementations are missing to trigger a link time error.
 /// Implementations for new devices, including the host, should go into a
 /// dedicated begin/end declare variant.
 ///
 ///{
 
void *&state::lookupPtr(ValueKind Kind, bool IsReadonly) {
   case state::VK_ParallelRegionFn:
     return TeamState.ParallelRegionFnVar;
   default:
     break;
   }
   __builtin_unreachable();
 }
 
-void state::init(bool IsSPMD) {
+void state::init(bool IsSPMD, KernelEnvironmentTy &KernelEnv) {
   SharedMemorySmartStack.init(IsSPMD);
   if (mapping::isInitialThreadInLevel0(IsSPMD)) {
     TeamState.init(IsSPMD);
     DebugEntryRAII::init();
+    KernelEnvironmentPtr = &KernelEnv;
   }
 
   ThreadStates[mapping::getThreadIdInBlock()] = nullptr;
 }
 
+KernelEnvironmentTy &state::getKernelEnvironment() {
+  return *KernelEnvironmentPtr;
+}
+
 void state::enterDataEnvironment() {
   unsigned TId = mapping::getThreadIdInBlock();
   ThreadStateTy *NewThreadState =
       static_cast<ThreadStateTy *>(__kmpc_alloc_shared(sizeof(ThreadStateTy)));
   NewThreadState->init(ThreadStates[TId]);
   ThreadStates[TId] = NewThreadState;
 }
 

openmp/libomptarget/deviceRTLs/common/src/omptarget.cu

do {
     }
 
     __kmpc_barrier_simple_spmd(Ident, TId);
 
   } while (true);
 }
 
 EXTERN
-int32_t __kmpc_target_init(ident_t *Ident, int8_t Mode,
-                           bool UseGenericStateMachine,
+int32_t __kmpc_target_init(KernelEnvironmentTy& KernelEnv,
                            bool RequiresFullRuntime) {
-  const bool IsSPMD = Mode & llvm::omp::OMP_TGT_EXEC_MODE_SPMD;
+  bool IsSPMD = KernelEnv.Configuration.ExecMode & llvm::omp::OMP_TGT_EXEC_MODE_SPMD;
+  bool UseGenericStateMachine = KernelEnv.Configuration.UseGenericStateMachine;
   int TId = __kmpc_get_hardware_thread_id_in_block();
   if (IsSPMD)
     __kmpc_spmd_kernel_init(RequiresFullRuntime);
   else
     __kmpc_generic_kernel_init();
 
   if (IsSPMD) {
-    __kmpc_barrier_simple_spmd(Ident, TId);
+    __kmpc_barrier_simple_spmd(&KernelEnv.Ident, TId);
     return -1;
   }
 
   if (TId == GetMasterThreadID())
     return -1;
 
   // Enter the generic state machine if enabled and if this thread can possibly
   // be an active worker thread.
   //
   // The latter check is important for NVIDIA Pascal (but not Volta) and AMD
   // GPU.  In those cases, a single thread can apparently satisfy a barrier on
   // behalf of all threads in the same warp.  Thus, it would not be safe for
   // other threads in the main thread's warp to reach the first
   // __kmpc_barrier_simple_spmd call in __kmpc_target_region_state_machine
   // before the main thread reaches its corresponding
   // __kmpc_barrier_simple_spmd call: that would permit all active worker
   // threads to proceed before the main thread has actually set
   // omptarget_nvptx_workFn, and then they would immediately quit without
   // doing any work.  GetNumberOfWorkersInTeam() does not include any of the
   // main thread's warp, so none of its threads can ever be active worker
   // threads.
   if (UseGenericStateMachine && TId < GetNumberOfWorkersInTeam())
-    __kmpc_target_region_state_machine(Ident);
+    __kmpc_target_region_state_machine(&KernelEnv.Ident);
 
   return TId;
 }
 
 EXTERN
-void __kmpc_target_deinit(ident_t *Ident, int8_t Mode,
-                          bool RequiresFullRuntime) {
-  const bool IsSPMD = Mode & llvm::omp::OMP_TGT_EXEC_MODE_SPMD;
+void __kmpc_target_deinit(bool RequiresFullRuntime) {
+  const bool IsSPMD = __kmpc_is_spmd_exec_mode();
   if (IsSPMD)
     __kmpc_spmd_kernel_deinit(RequiresFullRuntime);
   else
     __kmpc_generic_kernel_deinit();
 }
 
 #pragma omp end declare target

openmp/libomptarget/deviceRTLs/interface.h

EXTERN void __kmpc_taskloop(kmp_Ident *loc, uint32_t global_tid,
                             int32_t sched, uint64_t grainsize, void *task_dup);
 
 // cancel
 EXTERN int32_t __kmpc_cancellationpoint(kmp_Ident *loc, int32_t global_tid,
                                         int32_t cancelVal);
 EXTERN int32_t __kmpc_cancel(kmp_Ident *loc, int32_t global_tid,
                              int32_t cancelVal);
 
+#include "llvm/Frontend/OpenMP/KernelEnvironment.h"
 // non standard
-EXTERN int32_t __kmpc_target_init(ident_t *Ident, int8_t Mode,
-                                  bool UseGenericStateMachine,
-                                  bool RequiresFullRuntime);
-EXTERN void __kmpc_target_deinit(ident_t *Ident, int8_t Mode,
+EXTERN int32_t __kmpc_target_init(KernelEnvironmentTy& KernelEnvironment,
                                   bool RequiresFullRuntime);
+EXTERN void __kmpc_target_deinit(bool RequiresFullRuntime);
 EXTERN void __kmpc_kernel_prepare_parallel(void *WorkFn,
                                            int32_t NumThreadsClause);
 EXTERN bool __kmpc_kernel_parallel(void **WorkFn);
 EXTERN void __kmpc_kernel_end_parallel();
 
 EXTERN void __kmpc_data_sharing_init_stack();
 EXTERN void __kmpc_begin_sharing_variables(void ***GlobalArgs, size_t nArgs);
 EXTERN void __kmpc_end_sharing_variables();