[WIP][OpenMP][DeviceRTL] Add the initial support for SIMD execution (NOT FOR REVIEW)

[tianshilei1992]

rebase and set dependence

[tianshilei1992]

fix compilation errors

[tianshilei1992]

When it is target simd:

• outlined function can be emitted.
• function call to __kmpc_simd_51 can be emitted.

openmp/libomptarget/DeviceRTL/include/Interface.h

 
 /// TODO
 void __kmpc_serialized_parallel(IdentTy *Loc, uint32_t);
 
 /// TODO
 void __kmpc_end_serialized_parallel(IdentTy *Loc, uint32_t);
 
 /// TODO
+bool __kmpc_kernel_simd(SIMDRegionFnTy *WorkFn);
+
+/// TODO
+void __kmpc_kernel_end_simd();
+
+/// TODO
 void __kmpc_push_proc_bind(IdentTy *Loc, uint32_t TId, int ProcBind);
 
 /// TODO
 void __kmpc_push_num_teams(IdentTy *Loc, int32_t TId, int32_t NumTeams,
                            int32_t ThreadLimit);
 
 /// TODO
 uint16_t __kmpc_parallel_level(IdentTy *Loc, uint32_t);

openmp/libomptarget/DeviceRTL/include/Mapping.h

 
 #pragma omp declare target
 
 inline constexpr uint32_t MaxThreadsPerTeam = 1024;
 
 #pragma omp end declare target
 
 /// Initialize the mapping machinery.
-void init(bool IsSPMD);
+void init(int8_t Mode);
 
 /// Return true if the kernel is executed in SPMD mode.
 bool isSPMDMode();
 
 /// Return true if the kernel is executed in generic mode.
 bool isGenericMode();
 
+/// Return true if the kernel is executed in SIMD mode.
+bool isSIMDMode();
+
 /// Return true if the executing thread is the main thread in generic mode.
 bool isMainThreadInGenericMode();
 
 /// Return true if the executing thread has the lowest Id of the active threads
 /// in the warp.
 bool isLeaderInWarp();
 
 /// Return a mask describing all active threads in the warp.
 LaneMaskTy activemask();
 
 /// Return a mask describing all threads with a smaller Id in the warp.
 LaneMaskTy lanemaskLT();
 
 /// Return a mask describing all threads with a larget Id in the warp.
 LaneMaskTy lanemaskGT();
 
 /// Return the thread Id in the warp, in [0, getWarpSize()).
 uint32_t getThreadIdInWarp();
 
 /// Return the thread Id in the block, in [0, getBlockSize()).
 uint32_t getThreadIdInBlock();
 
+/// Return the logic thread Id, which depends on how we map an OpenMP thread to
+/// the target device. In non-SIMD mode, we map an OpenMP thread to a device
+/// thread. In SIMD mode, we map an OpenMP thread to a warp, and each thread in
+/// the warp is a SIMD lane.
+uint32_t getLogicThreadId();
+
 /// Return the warp id in the block.
 uint32_t getWarpId();
 
 /// Return the warp size, thus number of threads in the warp.
 uint32_t getWarpSize();
 
 /// Return the number of warps in the block.
 uint32_t getNumberOfWarpsInBlock();
 
 /// Return the block Id in the kernel, in [0, getKernelSize()).
 uint32_t getBlockId();
 
 /// Return the block size, thus number of threads in the block.
 uint32_t getBlockSize();
 
 /// Return the number of blocks in the kernel.
 uint32_t getNumberOfBlocks();
 
 /// Return the kernel size, thus number of threads in the kernel.
 uint32_t getKernelSize();
 
 /// Return the number of processing elements on the device.
 uint32_t getNumberOfProcessorElements();
 
+namespace utils {
+/// Return true if \p Mode indicates SPMD mode.
+inline bool isSPMDMode(int8_t Mode) { return Mode & OMP_TGT_EXEC_MODE_SPMD; }
+
+/// Return true if \p Mode indicates generic mode.
+inline bool isGenericMode(int8_t Mode) {
+  return Mode & OMP_TGT_EXEC_MODE_GENERIC;
+}
+
+/// Return true if \p Mode indicates SIMD mode.
+inline bool isSIMDMode(int8_t Mode) { return Mode & OMP_TGT_EXEC_MODE_SIMD; }
+} // namespace utils
+
 } // namespace mapping
 
 } // namespace _OMP
 
 #endif

openmp/libomptarget/DeviceRTL/include/State.h

 #include "Types.h"
 
 #pragma omp declare target
 
 namespace _OMP {
 
 namespace state {
 
 inline constexpr uint32_t SharedScratchpadSize = SHARED_SCRATCHPAD_SIZE;
 
 /// Initialize the state machinery. Must be called by all threads.
-void init(bool IsSPMD);
+void init(int8_t Mode);
 
 /// TODO
 enum ValueKind {
   VK_NThreads,
   VK_Level,
   VK_ActiveLevel,
   VK_MaxActiveLevels,
   VK_RunSched,
   // ---
   VK_RunSchedChunk,
   VK_ParallelRegionFn,
   VK_ParallelTeamSize,
+  // SIMD
+  VK_SIMDLevel,
+  VK_SIMDRegionFn,
+  VK_SIMDLaneWidth,
 };
 
 /// TODO
 void enterDataEnvironment();
 
 /// TODO
 void exitDataEnvironment();
 

 
 /// TODO
 inline state::Value<uint32_t, state::VK_ParallelTeamSize> ParallelTeamSize;
 
 /// TODO
 inline state::PtrValue<ParallelRegionFnTy, state::VK_ParallelRegionFn>
     ParallelRegionFn;
 
+/// TODO
+inline state::Value<uint32_t, state::VK_SIMDLaneWidth> SIMDLaneWidth;
+
+/// TODO
+inline state::PtrValue<SIMDRegionFnTy, state::VK_SIMDRegionFn> SIMDRegionFn;
+
 void runAndCheckState(void(Func(void)));
 
 void assumeInitialState(bool IsSPMD);
 
+/// Propagate the thread state from the leader in the warp to the rest of SIMD
+/// workers. This function should only be called in SIMD mode.
+void propagateThreadState(unsigned SIMDLen);
+
 } // namespace state
 
 namespace icv {
 
 /// TODO
 inline state::Value<uint32_t, state::VK_NThreads> NThreads;
 
 /// TODO
 inline state::Value<uint32_t, state::VK_Level> Level;
 
 /// The `active-level` describes which of the parallel level counted with the
 /// `level-var` is active. There can only be one.
 ///
 /// active-level-var is 1, if ActiveLevelVar is not 0, otherweise it is 0.
 inline state::Value<uint32_t, state::VK_ActiveLevel> ActiveLevel;
 
 /// TODO
 inline state::Value<uint32_t, state::VK_MaxActiveLevels> MaxActiveLevels;
 
 /// TODO
 inline state::Value<uint32_t, state::VK_RunSched> RunSched;
 
+/// TODO
+inline state::Value<uint32_t, state::VK_SIMDLevel> SIMDLevel;
+
 } // namespace icv
 
 namespace memory {
 
 /// Alloca \p Size bytes in shared memory, if possible, for \p Reason.
 ///
 /// Note: See the restrictions on __kmpc_alloc_shared for proper usage.
 void *allocShared(uint64_t Size, const char *Reason);

openmp/libomptarget/DeviceRTL/include/Synchronization.h

 
 #include "Types.h"
 
 namespace _OMP {
 
 namespace synchronize {
 
 /// Initialize the synchronization machinery. Must be called by all threads.
-void init(bool IsSPMD);
+void init(int8_t Mode);
 
 /// Synchronize all threads in a warp identified by \p Mask.
 void warp(LaneMaskTy Mask);
 
 /// Synchronize all threads in a block.
 void threads();
 
 } // namespace synchronize

openmp/libomptarget/DeviceRTL/include/Types.h

char const *psource; /**<  String describing the source location.
                        which describe the source file, the function and a pair
                        of line numbers that delimit the construct. */
 };
 
 using __kmpc_impl_lanemask_t = LaneMaskTy;
 
 using ParallelRegionFnTy = void *;
 
+using SIMDRegionFnTy = void *;
+
 using CriticalNameTy = int32_t[8];
 
 struct omp_lock_t {
   void *Lock;
 };
 
 using InterWarpCopyFnTy = void (*)(void *src, int32_t warp_num);
 using ShuffleReductFnTy = void (*)(void *rhsData, int16_t lane_id,
typedef enum omp_allocator_handle_t {
   omp_pteam_mem_alloc = 7,
   omp_thread_mem_alloc = 8,
   KMP_ALLOCATOR_MAX_HANDLE = ~(0U)
 } omp_allocator_handle_t;
 
 enum OMPTgtExecModeFlags : int8_t {
   OMP_TGT_EXEC_MODE_GENERIC = 1 << 0,
   OMP_TGT_EXEC_MODE_SPMD = 1 << 1,
+  OMP_TGT_EXEC_MODE_SIMD = 1 << 2,
 };
 
 #define __PRAGMA(STR) _Pragma(#STR)
 #define OMP_PRAGMA(STR) __PRAGMA(omp STR)
 
 #define SHARED(NAME)                                                           \
   NAME [[clang::loader_uninitialized]];                                        \
   OMP_PRAGMA(allocate(NAME) allocator(omp_pteam_mem_alloc))

openmp/libomptarget/DeviceRTL/src/Kernel.cpp

 //===--- Kernel.cpp - OpenMP device kernel interface -------------- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file contains the kernel entry points for the device.
 //
 //===----------------------------------------------------------------------===//
 
 #include "Debug.h"
 #include "Interface.h"
 #include "Mapping.h"
 #include "State.h"
 #include "Synchronization.h"
 #include "Types.h"
 
 using namespace _OMP;
 
 #pragma omp declare target
 
-static void inititializeRuntime(bool IsSPMD) {
+static void inititializeRuntime(int Mode) {
   // Order is important here.
-  synchronize::init(IsSPMD);
-  mapping::init(IsSPMD);
-  state::init(IsSPMD);
+  synchronize::init(Mode);
+  mapping::init(Mode);
+  state::init(Mode);
 }
 
 /// Simple generic state machine for worker threads.
 static void genericStateMachine(IdentTy *Ident) {
 
-  uint32_t TId = mapping::getThreadIdInBlock();
+  uint32_t TId = mapping::getLogicThreadId();
 
   do {
     ParallelRegionFnTy WorkFn = 0;
 
     // Wait for the signal that we have a new work function.
     synchronize::threads();
 
     // Retrieve the work function from the runtime.
if (IsActive) {
       __kmpc_kernel_end_parallel();
     }
 
     synchronize::threads();
 
   } while (true);
 }
 
+namespace {
+void runSIMDStateMachine(IdentTy *Ident) {
+  uint32_t LaneId = mapping::getThreadIdInWarp();
+  do {
+    SIMDRegionFnTy WorkFn = nullptr;
+
+    // Wait for the signal that we have a new work function.
+    synchronize::warp(mapping::activemask());
+
+    // Retrieve the work function from the runtime.
+    bool IsActive = __kmpc_kernel_simd(&WorkFn);
+
+    if (!WorkFn)
+      return;
+
+    if (IsActive) {
+      ((void (*)(uint32_t, uint32_t))WorkFn)(0, LaneId);
+      __kmpc_kernel_end_simd();
+    }
+
+    synchronize::warp(mapping::activemask());
+  } while (true);
+}
+} // namespace
+
 extern "C" {
 
 /// Initialization
 ///
 /// \param Ident               Source location identification, can be NULL.
 ///
 int32_t __kmpc_target_init(IdentTy *Ident, int8_t Mode,
                            bool UseGenericStateMachine, bool) {
-  const bool IsSPMD = Mode & OMP_TGT_EXEC_MODE_SPMD;
-  if (IsSPMD) {
-    inititializeRuntime(/* IsSPMD */ true);
-    synchronize::threads();
-  } else {
-    inititializeRuntime(/* IsSPMD */ false);
-    // No need to wait since only the main threads will execute user
-    // code and workers will run into a barrier right away.
+  inititializeRuntime(Mode);
+
+  // For all SIMD workers, start the simd state machine.
+  if (mapping::utils::isSIMDMode(Mode)) {
+    uint32_t LaneId = mapping::getThreadIdInWarp();
+    if (LaneId) {
+      runSIMDStateMachine(Ident);
+      return LaneId;
+    }
   }
 
+  const bool IsSPMD = mapping::utils::isSPMDMode(Mode);
+  if (IsSPMD)
+    synchronize::threads();
+
   if (IsSPMD) {
     state::assumeInitialState(IsSPMD);
     return -1;
   }
 
   if (mapping::isMainThreadInGenericMode())
     return -1;
 
   if (UseGenericStateMachine)
     genericStateMachine(Ident);
 
   return mapping::getThreadIdInBlock();
 }
 
 /// De-Initialization
 ///
 /// In non-SPMD, this function releases the workers trapped in a state machine
 /// and also any memory dynamically allocated by the runtime.
 ///
 /// \param Ident Source location identification, can be NULL.
 ///
 void __kmpc_target_deinit(IdentTy *Ident, int8_t Mode, bool) {
-  const bool IsSPMD = Mode & OMP_TGT_EXEC_MODE_SPMD;
+  const bool IsSPMD = mapping::utils::isSPMDMode(Mode);
+
   state::assumeInitialState(IsSPMD);
   if (IsSPMD)
     return;
 
   // Signal the workers to exit the state machine and exit the kernel.
   state::ParallelRegionFn = nullptr;
 }
 
 int8_t __kmpc_is_spmd_exec_mode() { return mapping::isSPMDMode(); }
 }
 
 #pragma omp end declare target

openmp/libomptarget/DeviceRTL/src/Mapping.cpp

 //===------- Mapping.cpp - OpenMP device runtime mapping helpers -- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 //
 //===----------------------------------------------------------------------===//
 
 #include "Mapping.h"
 #include "State.h"
 #include "Types.h"
 #include "Utils.h"
 
 #pragma omp declare target
 
 #include "llvm/Frontend/OpenMP/OMPGridValues.h"
 
bool mapping::isMainThreadInGenericMode() {
   uint32_t MainTId = (mapping::getNumberOfProcessorElements() - 1) &
                      ~(mapping::getWarpSize() - 1);
   return mapping::getThreadIdInBlock() == MainTId;
 }
 
 bool mapping::isLeaderInWarp() {
   __kmpc_impl_lanemask_t Active = mapping::activemask();
   __kmpc_impl_lanemask_t LaneMaskLT = mapping::lanemaskLT();
-  return utils::popc(Active & LaneMaskLT) == 0;
+  return ::_OMP::utils::popc(Active & LaneMaskLT) == 0;
 }
 
 LaneMaskTy mapping::activemask() { return impl::activemask(); }
 
 LaneMaskTy mapping::lanemaskLT() { return impl::lanemaskLT(); }
 
 LaneMaskTy mapping::lanemaskGT() { return impl::lanemaskGT(); }
 
 uint32_t mapping::getThreadIdInWarp() { return impl::getThreadIdInWarp(); }
 
 uint32_t mapping::getThreadIdInBlock() { return impl::getThreadIdInBlock(); }
 
+uint32_t mapping::getLogicThreadId() {
+  if (mapping::isSIMDMode())
+    return mapping::getWarpId();
+
+  return mapping::getThreadIdInBlock();
+}
+
 uint32_t mapping::getBlockSize() { return impl::getBlockSize(); }
 
 uint32_t mapping::getKernelSize() { return impl::getKernelSize(); }
 
 uint32_t mapping::getBlockId() { return impl::getBlockId(); }
 
 uint32_t mapping::getNumberOfBlocks() { return impl::getNumberOfBlocks(); }
 
 uint32_t mapping::getNumberOfProcessorElements() {
   return impl::getNumberOfProcessorElements();
 }
 
 uint32_t mapping::getWarpId() { return impl::getWarpId(); }
 
 uint32_t mapping::getWarpSize() { return impl::getWarpSize(); }
 
 uint32_t mapping::getNumberOfWarpsInBlock() {
   return impl::getNumberOfWarpsInBlock();
 }
 
 /// Execution mode
 ///
 ///{
-static int SHARED(IsSPMDMode);
+static int8_t SHARED(ExecutionMode);
 
-void mapping::init(bool IsSPMD) {
+void mapping::init(int8_t Mode) {
   if (!mapping::getThreadIdInBlock())
-    IsSPMDMode = IsSPMD;
+    ExecutionMode = Mode;
 }
 
-bool mapping::isSPMDMode() { return IsSPMDMode; }
+bool mapping::isSPMDMode() { return mapping::utils::isSPMDMode(ExecutionMode); }
+
+bool mapping::isGenericMode() {
+  return mapping::utils::isGenericMode(ExecutionMode);
+}
 
-bool mapping::isGenericMode() { return !isSPMDMode(); }
+bool mapping::isSIMDMode() { return mapping::utils::isSIMDMode(ExecutionMode); }
 ///}
 
 extern "C" {
 __attribute__((noinline)) uint32_t __kmpc_get_hardware_thread_id_in_block() {
   return mapping::getThreadIdInBlock();
 }
 }
 #pragma omp end declare target

openmp/libomptarget/DeviceRTL/src/Parallelism.cpp

 //    syncthread A. Thus they must preserve their current threadId that
 //    is larger than thread in team.
 //
 //    To make a long story short...
 //
 //===----------------------------------------------------------------------===//
 
 #include "Debug.h"
 #include "Interface.h"
 #include "Mapping.h"
 #include "State.h"
 #include "Synchronization.h"
 #include "Types.h"
 #include "Utils.h"
 
 using namespace _OMP;
 
 #pragma omp declare target
 
 namespace {
 
 uint32_t determineNumberOfThreads(int32_t NumThreadsClause) {
   uint32_t NThreadsICV =
       NumThreadsClause != -1 ? NumThreadsClause : icv::NThreads;
-  uint32_t NumThreads = mapping::getBlockSize();
+
+  const bool IsSIMDMode = mapping::isSIMDMode();
+
+  uint32_t NumThreads =
+      IsSIMDMode ? mapping::getNumberOfWarpsInBlock() : mapping::getBlockSize();
 
   if (NThreadsICV != 0 && NThreadsICV < NumThreads)
     NumThreads = NThreadsICV;
 
   // Round down to a multiple of WARPSIZE since it is legal to do so in OpenMP.
+  // We don't need this for SIMD mode because an OpenMP thread is mapped to a
+  // warp on the device and it can be any number.
+  if (!IsSIMDMode) {
     if (NumThreads < mapping::getWarpSize())
       NumThreads = 1;
     else
       NumThreads = (NumThreads & ~((uint32_t)mapping::getWarpSize() - 1));
+  }
 
   return NumThreads;
 }
 
+uint32_t determineSIMDLen(int32_t SIMDLen, int32_t SafeLen) {
+  ASSERT(mapping::isSIMDMode());
+
+  // TODO: This is probably not right if the schedule is different.
+  if (SafeLen < SIMDLen)
+    SIMDLen = SafeLen;
+
+  // We currently maps an OpenMP thread to a warp in SIMD mode. If the simdlen
+  // is larger than the warp size, we have to ceil it.
+  if (SIMDLen > mapping::getWarpSize())
+    SIMDLen = mapping::getWarpSize();
+
+  return SIMDLen;
+}
+
 // Invoke an outlined parallel function unwrapping arguments (up to 32).
 void invokeMicrotask(int32_t global_tid, int32_t bound_tid, void *fn,
                      void **args, int64_t nargs) {
   switch (nargs) {
 #include "generated_microtask_cases.gen"
   default:
     PRINT("Too many arguments in kmp_invoke_microtask, aborting execution.\n");
     __builtin_trap();
   }
 }
 
 } // namespace
 
 extern "C" {
 
+void __kmpc_simd_51(IdentTy *ident, int32_t, int32_t if_expr, int32_t safelen,
+                    int32_t simdlen, int order, void *fn, void *wrapper_fn,
+                    void **args, int64_t nargs) {
+  // Handle non-SIMD case first, which can be:
+  // - if clause is evaluted to false
+  // - simdlen is set to 1
+  // - it is already in simd region
+  const uint32_t LogicThreadId = mapping::getLogicThreadId();
+  if (OMP_UNLIKELY(!if_expr || simdlen == 1 || safelen == 1 ||
+                   icv::SIMDLevel)) {
+    invokeMicrotask(LogicThreadId, 0, fn, args, nargs);
+    return;
+  }
+
+  // Only the leader of each warp can execute the following code.
+  ASSERT(mapping::isLeaderInWarp());
+
+  const uint32_t SIMDLen = determineSIMDLen(simdlen, safelen);
+
+  if (LogicThreadId == 0)
+    state::SIMDLaneWidth = SIMDLen;
+
+  // Propagates the thread state to all SIMD workers from the leader.
+  state::propagateThreadState(SIMDLen);
+
+  // Synchronize all threads (leaders).
+  synchronize::threads();
+
+  {
+    state::ValueRAII SIMDRegionFnRAII(state::SIMDRegionFn, wrapper_fn,
+                                      (void *)nullptr, true);
+    state::ValueRAII SIMDLevelRAII(icv::SIMDLevel, 1u, 0u, true);
+
+    // Signal SIMD workers
+    synchronize::warp(mapping::activemask());
+
+    // TODO: Leader in warp also has to execute the SIMD region.
+    // What we need:
+    // - A work-sharing function that can take both thread id and lane id into
+    // consideration.
+
+    // Synchronize after execution of the SIMD region.
+    synchronize::warp(mapping::activemask());
+  }
+}
+
 void __kmpc_parallel_51(IdentTy *ident, int32_t, int32_t if_expr,
                         int32_t num_threads, int proc_bind, void *fn,
                         void *wrapper_fn, void **args, int64_t nargs) {
 
-  uint32_t TId = mapping::getThreadIdInBlock();
+  uint32_t TId = mapping::getLogicThreadId();
   // Handle the serialized case first, same for SPMD/non-SPMD.
   if (OMP_UNLIKELY(!if_expr || icv::Level)) {
     __kmpc_serialized_parallel(ident, TId);
     invokeMicrotask(TId, 0, fn, args, nargs);
     __kmpc_end_serialized_parallel(ident, TId);
     return;
   }
 
__kmpc_kernel_parallel(ParallelRegionFnTy *WorkFn) {
   // Work function and arguments for L1 parallel region.
   *WorkFn = state::ParallelRegionFn;
 
   // If this is the termination signal from the master, quit early.
   if (!*WorkFn)
     return false;
 
   // Set to true for workers participating in the parallel region.
-  uint32_t TId = mapping::getThreadIdInBlock();
+  uint32_t TId = mapping::getLogicThreadId();
   bool ThreadIsActive = TId < state::ParallelTeamSize;
   return ThreadIsActive;
 }
 
 __attribute__((noinline)) void __kmpc_kernel_end_parallel() {
   // In case we have modified an ICV for this thread before a ThreadState was
   // created. We drop it now to not contaminate the next parallel region.
   ASSERT(!mapping::isSPMDMode());
   uint32_t TId = mapping::getThreadIdInBlock();
   state::resetStateForThread(TId);
   ASSERT(!mapping::isSPMDMode());
 }
 
+__attribute__((noinline)) bool __kmpc_kernel_simd(SIMDRegionFnTy *WorkFn) {
+  // Work function and arguments for L1 SIMD region.
+  *WorkFn = state::SIMDRegionFn;
+
+  // If this is the termination signal from the master, quit early.
+  if (!*WorkFn)
+    return false;
+
+  // Set to true for workers participating in the parallel region.
+  uint32_t LaneId = mapping::getThreadIdInWarp();
+  bool LaneActive = LaneId < state::SIMDLaneWidth;
+  return LaneActive;
+}
+
+__attribute__((noinline)) void __kmpc_kernel_end_simd() {
+  // TODO: Some clean-up of SIMD execution
+}
+
 void __kmpc_serialized_parallel(IdentTy *, uint32_t TId) {
   state::enterDataEnvironment();
   ++icv::Level;
 }
 
 void __kmpc_end_serialized_parallel(IdentTy *, uint32_t TId) {
   state::exitDataEnvironment();
   --icv::Level;

openmp/libomptarget/DeviceRTL/src/State.cpp

 //===------ State.cpp - OpenMP State & ICV interface ------------- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 //===----------------------------------------------------------------------===//
 
 #include "State.h"
 #include "Configuration.h"
 #include "Debug.h"
 #include "Interface.h"
 #include "Mapping.h"
 #include "Synchronization.h"
 #include "Types.h"
 #include "Utils.h"
 
void ICVStateTy::assertEqual(const ICVStateTy &Other) const {
   ASSERT(ActiveLevelVar == Other.ActiveLevelVar);
   ASSERT(MaxActiveLevelsVar == Other.MaxActiveLevelsVar);
   ASSERT(RunSchedVar == Other.RunSchedVar);
   ASSERT(RunSchedChunkVar == Other.RunSchedChunkVar);
 }
 
 struct TeamStateTy {
   /// TODO: provide a proper init function.
-  void init(bool IsSPMD);
+  void init(int Mode);
 
   bool operator==(const TeamStateTy &) const;
 
   void assertEqual(TeamStateTy &Other) const;
 
   /// ICVs
   ///
   /// Preallocated storage for ICV values that are used if the threads have not
   /// set a custom default. The latter is supported but unlikely and slow(er).
   ///
   ///{
   ICVStateTy ICVState;
   ///}
 
   uint32_t ParallelTeamSize;
   ParallelRegionFnTy ParallelRegionFnVar;
 };
 
 TeamStateTy SHARED(TeamState);
 
-void TeamStateTy::init(bool IsSPMD) {
+void TeamStateTy::init(int Mode) {
+  // In SIMD mode, we map an OpenMP thread to a warp.
+  if (mapping::utils::isSIMDMode(Mode))
+    ICVState.NThreadsVar = mapping::getNumberOfWarpsInBlock();
+  else
     ICVState.NThreadsVar = mapping::getBlockSize();
+
   ICVState.LevelVar = 0;
   ICVState.ActiveLevelVar = 0;
   ICVState.MaxActiveLevelsVar = 1;
   ICVState.RunSchedVar = omp_sched_static;
   ICVState.RunSchedChunkVar = 1;
   ParallelTeamSize = 1;
   ParallelRegionFnVar = nullptr;
 }
void *&state::lookupPtr(ValueKind Kind, bool IsReadonly) {
   case state::VK_ParallelRegionFn:
     return TeamState.ParallelRegionFnVar;
   default:
     break;
   }
   __builtin_unreachable();
 }
 
-void state::init(bool IsSPMD) {
+void state::init(int8_t Mode) {
+  const bool IsSPMD = mapping::utils::isSPMDMode(Mode);
   SharedMemorySmartStack.init(IsSPMD);
   if (!mapping::getThreadIdInBlock())
     TeamState.init(IsSPMD);
 
   ThreadStates[mapping::getThreadIdInBlock()] = nullptr;
 }
 
 void state::enterDataEnvironment() {

 void state::assumeInitialState(bool IsSPMD) {
   TeamStateTy InitialTeamState;
   InitialTeamState.init(IsSPMD);
   InitialTeamState.assertEqual(TeamState);
   ASSERT(!ThreadStates[mapping::getThreadIdInBlock()]);
   ASSERT(mapping::isSPMDMode() == IsSPMD);
 }
 
+void state::propagateThreadState(unsigned SIMDLen) {
+  ASSERT(mapping::isSIMDMode());
+  ASSERT(mapping::isLeaderInWarp());
+
+  const uint32_t TId = mapping::getThreadIdInBlock();
+  for (int I = 1; I < SIMDLen; ++I)
+    ThreadStates[I + TId] = ThreadStates[TId];
+}
+
 extern "C" {
 void omp_set_dynamic(int V) {}
 
 int omp_get_dynamic(void) { return 0; }
 
 void omp_set_num_threads(int V) { icv::NThreads = V; }
 
 int omp_get_max_threads(void) { return icv::NThreads; }

 }
 
 void omp_set_schedule(omp_sched_t ScheduleKind, int ChunkSize) {
   icv::RunSched = (int)ScheduleKind;
   state::RunSchedChunk = ChunkSize;
 }
 
 int omp_get_ancestor_thread_num(int Level) {
-  return returnValIfLevelIsActive(Level, mapping::getThreadIdInBlock(), 0);
+  return returnValIfLevelIsActive(Level, mapping::getLogicThreadId(), 0);
 }
 
 int omp_get_thread_num(void) {
   return omp_get_ancestor_thread_num(omp_get_level());
 }
 
 int omp_get_team_size(int Level) {
   return returnValIfLevelIsActive(Level, state::ParallelTeamSize, 1);

openmp/libomptarget/DeviceRTL/src/Synchronization.cpp

 //===- Synchronization.cpp - OpenMP Device synchronization API ---- c++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // Include all synchronization.
 //
 //===----------------------------------------------------------------------===//
 
 #include "Synchronization.h"
 
 #include "Debug.h"
 #include "Interface.h"
 #include "Mapping.h"
 #include "State.h"
 #include "Types.h"
 #include "Utils.h"
 
void setLock(omp_lock_t *Lock) {
   } // wait for 0 to be the read value
 }
 
 #pragma omp end declare variant
 ///}
 
 } // namespace impl
 
-void synchronize::init(bool IsSPMD) {
-  if (!IsSPMD)
+void synchronize::init(int8_t Mode) {
+  if (!mapping::utils::isSPMDMode(Mode) || mapping::utils::isSIMDMode(Mode))
     impl::namedBarrierInit();
 }
 
 void synchronize::warp(LaneMaskTy Mask) { impl::syncWarp(Mask); }
 
 void synchronize::threads() { impl::syncThreads(); }
 
 void fence::team(int Ordering) { impl::fenceTeam(Ordering); }

openmp/libomptarget/DeviceRTL/src/Workshare.cpp

 //===----- Workshare.cpp -  OpenMP workshare implementation ------ C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file contains the implementation of the KMPC interface
 // for the loop construct plus other worksharing constructs that use the same
 // interface as loops.
 //
 //===----------------------------------------------------------------------===//
 
 #include "Debug.h"
 #include "Interface.h"
 #include "Mapping.h"
 #include "State.h"
 #include "Synchronization.h"
 #include "Types.h"
 #include "Utils.h"
 
 using namespace _OMP;
 
template <typename T, typename ST> struct omptarget_nvptx_LoopSupport {
   static int OrderedSchedule(kmp_sched_t schedule) {
     return schedule >= kmp_sched_ordered_first &&
            schedule <= kmp_sched_ordered_last;
   }
 
   static void dispatch_init(IdentTy *loc, int32_t threadId,
                             kmp_sched_t schedule, T lb, T ub, ST st, ST chunk,
                             DynamicScheduleTracker *DST) {
-    int tid = mapping::getThreadIdInBlock();
+    int tid = mapping::getLogicThreadId();
     T tnum = omp_get_num_threads();
     T tripCount = ub - lb + 1; // +1 because ub is inclusive
     ASSERT0(LT_FUSSY, threadId < tnum,
             "current thread is not needed here; error");
 
     /* Currently just ignore the monotonic and non-monotonic modifiers
      * (the compiler isn't producing them * yet anyway).
      * When it is we'll want to look at them somewhere here and use that

openmp/libomptarget/plugins/cuda/src/rtl.cpp

 //===----RTLs/cuda/src/rtl.cpp - Target RTLs Implementation ------- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // RTL for CUDA machine
 //
 //===----------------------------------------------------------------------===//
 
 #include <cassert>
 #include <cstddef>
 #include <cuda.h>
 #include <list>
 #include <memory>
 #include <mutex>
 #include <string>
 #include <unordered_map>
 #include <vector>
 
 #include "Debug.h"
int runTargetTeamRegion(const int DeviceId, void *TgtEntryPtr, void **TgtArgs,
     for (int I = 0; I < ArgNum; ++I) {
       Ptrs[I] = (void *)((intptr_t)TgtArgs[I] + TgtOffsets[I]);
       Args[I] = &Ptrs[I];
     }
 
     KernelTy *KernelInfo = reinterpret_cast<KernelTy *>(TgtEntryPtr);
 
     const bool IsSPMDGenericMode =
-        KernelInfo->ExecutionMode == llvm::omp::OMP_TGT_EXEC_MODE_GENERIC_SPMD;
+        KernelInfo->ExecutionMode & llvm::omp::OMP_TGT_EXEC_MODE_GENERIC_SPMD;
     const bool IsSPMDMode =
-        KernelInfo->ExecutionMode == llvm::omp::OMP_TGT_EXEC_MODE_SPMD;
+        !IsSPMDGenericMode &&
+        (KernelInfo->ExecutionMode & llvm::omp::OMP_TGT_EXEC_MODE_SPMD);
     const bool IsGenericMode =
-        KernelInfo->ExecutionMode == llvm::omp::OMP_TGT_EXEC_MODE_GENERIC;
+        !IsSPMDGenericMode &&
+        (KernelInfo->ExecutionMode & llvm::omp::OMP_TGT_EXEC_MODE_GENERIC);
+    const bool IsSIMDMode =
+        KernelInfo->ExecutionMode & llvm::omp::OMP_TGT_EXEC_MODE_SIMD;
 
     int CudaThreadsPerBlock;
     if (ThreadLimit > 0) {
+      if (IsSIMDMode) {
+        DP("Setting CUDA threads per block to requested %d\n",
+           ThreadLimit * DeviceData[DeviceId].WarpSize);
+        CudaThreadsPerBlock = ThreadLimit * DeviceData[DeviceId].WarpSize;
+      } else {
         DP("Setting CUDA threads per block to requested %d\n", ThreadLimit);
         CudaThreadsPerBlock = ThreadLimit;
         // Add master warp if necessary
         if (IsGenericMode) {
-        DP("Adding master warp: +%d threads\n", DeviceData[DeviceId].WarpSize);
+          DP("Adding master warp: +%d threads\n",
+             DeviceData[DeviceId].WarpSize);
           CudaThreadsPerBlock += DeviceData[DeviceId].WarpSize;
         }
+      }
     } else {
       DP("Setting CUDA threads per block to default %d\n",
          DeviceData[DeviceId].NumThreads);
       CudaThreadsPerBlock = DeviceData[DeviceId].NumThreads;
     }
 
     if (CudaThreadsPerBlock > DeviceData[DeviceId].ThreadsPerBlock) {
       DP("Threads per block capped at device limit %d\n",