[libomptarget] Implement wavefront functions for amdgcn

[JonChesterfield]

Summary

[libomptarget] Implement wavefront functions for amdgcn

[JonChesterfield]

Fills in some of the unresolved symbols for amdgcn target_impl with intrinsic calls. A bit architecture specific, but that seems OK in the context.

[jdoerfert]

The intrinsics and the computation looks a bit like black magic to me.

@arsenm Could you comment on this?

openmp/libomptarget/deviceRTLs/amdgcn/src/target_impl.hip

54

Is the argument unused on purpose? Why do we have it?

[arsenm]

openmp/libomptarget/deviceRTLs/amdgcn/src/target_impl.hip

21–22

What about wave32?

34

Wave32? WAVESIZE - 1?

[JonChesterfield]

openmp/libomptarget/deviceRTLs/amdgcn/src/target_impl.hip

21–22

Hi Matt. I didn't think we had any 32 bit warp / wavefront hardware support in tree. If we do there's some changes needed elsewhere in this library as well

54

It's used to pick out lanes to synchronise on nvptx (if I understand their documentation correctly). The lanes move in lockstep on amdgcn so I don't think there's any corresponding synchronisation to do here

[JonChesterfield]

openmp/libomptarget/deviceRTLs/amdgcn/src/target_impl.hip

21–22

Ah. gfx1010 has some code in clang, e.g. https://reviews.llvm.org/D63209

I'd still like to land this patch as it progresses support for most amdgcn.

I'll also start the task of supporting wave32. The complicated part is probably detecting what a given arch uses early enough to pick out the right typedef for lanemask_t.

[arsenm]

openmp/libomptarget/deviceRTLs/amdgcn/src/target_impl.hip

21–22

Putting off wave32 is fine, but might as well fix the hardcoded 63 here anyway

[JonChesterfield]

• 63 => warpsize-1

[JonChesterfield]

Ping

[jdoerfert]

I'm OK with, @arsenm?

[arsenm]

openmp/libomptarget/deviceRTLs/amdgcn/src/target_impl.hip

21

Capitalize comment

30

Capitalize comment

[JonChesterfield]

• initialise comments

openmp/libomptarget/deviceRTLs/amdgcn/src/target_impl.hip

 //===------- target_impl.hip - AMDGCN OpenMP GPU implementation --- HIP -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // Definitions of target specific functions
 //
 //===----------------------------------------------------------------------===//
 
 #include "target_impl.h"
 
+// Implementations initially derived from hcc
+
+static DEVICE uint32_t getLaneId(void) {
+  return __builtin_amdgcn_mbcnt_hi(~0u, __builtin_amdgcn_mbcnt_lo(~0u, 0u));
+}
+
+// Initialized with a 64-bit mask with bits set in positions less than the
+// thread's lane number in the warp
+DEVICE __kmpc_impl_lanemask_t __kmpc_impl_lanemask_lt() {
+  uint32_t lane = getLaneId();
+  int64_t ballot = __kmpc_impl_activemask();
+  uint64_t mask = ((uint64_t)1 << lane) - (uint64_t)1;
+  return mask & ballot;
+}
+
+// Initialized with a 64-bit mask with bits set in positions greater than the
+// thread's lane number in the warp
+DEVICE __kmpc_impl_lanemask_t __kmpc_impl_lanemask_gt() {
+  uint32_t lane = getLaneId();
+  if (lane == (WARPSIZE - 1))
+    return 0;
+  uint64_t ballot = __kmpc_impl_activemask();
+  uint64_t mask = (~((uint64_t)0)) << (lane + 1);
+  return mask & ballot;
+}
+
 DEVICE double __kmpc_impl_get_wtick() { return ((double)1E-9); }
 
 EXTERN uint64_t __clock64();
 DEVICE double __kmpc_impl_get_wtime() {
   return ((double)1.0 / 745000000.0) * __clock64();
 }
 
+// Warp vote function
+DEVICE __kmpc_impl_lanemask_t __kmpc_impl_activemask() {
+  // 33 is ICMP_NE from llvm/include/llvm/IR/InstrTypes.h
+  return __builtin_amdgcn_uicmp(1, 0, 33);
+}
+
+DEVICE int32_t __kmpc_impl_shfl_sync(__kmpc_impl_lanemask_t, int32_t var,
+                                     int32_t srcLane) {
+  int width = WARPSIZE;
+  int self = getLaneId();
+  int index = srcLane + (self & ~(width - 1));
+  return __builtin_amdgcn_ds_bpermute(index << 2, var);
+}
+
+DEVICE int32_t __kmpc_impl_shfl_down_sync(__kmpc_impl_lanemask_t, int32_t var,
+                                          uint32_t laneDelta, int32_t width) {
+  int self = getLaneId();
+  int index = self + laneDelta;
+  index = (int)(laneDelta + (self & (width - 1))) >= width ? self : index;
+  return __builtin_amdgcn_ds_bpermute(index << 2, var);
+}
+
 EXTERN uint64_t __ockl_get_local_size(uint32_t);
 EXTERN uint64_t __ockl_get_num_groups(uint32_t);
 DEVICE int GetNumberOfBlocksInKernel() { return __ockl_get_num_groups(0); }
 DEVICE int GetNumberOfThreadsInBlock() { return __ockl_get_local_size(0); }