[OpenMP][AMDGPU] Support of cov5 in the next gen plugin

[saiislam]

Summary

Initial attempt to support cov5 in the next gen plugin.

[jdoerfert]

Thanks for updating the new plugin too. Some comments below.

openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp

1666

Is this a per-kernel property or a per image property? If it's the latter, maybe add a TODO to move it to a image abstraction once we have a AMD specific one.

2134

COV is AMD-GPU specific I assume, right? If so, put the implementation in here and a default impl into the generic part. Also, add documentation to each new method, please.

openmp/libomptarget/plugins-nextgen/amdgpu/utils/UtilitiesRTL.h

29

I think we want to keep the layout, eventually we'll need it, right?

[saiislam]

Not required after D139730

openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp

private:
   hsa_executable_t Executable;
   hsa_code_object_t CodeObject;
 };
 
 /// Class implementing the AMDGPU kernel functionalities which derives from the
 /// generic kernel class.
 struct AMDGPUKernelTy : public GenericKernelTy {
   /// Create an AMDGPU kernel with a name and an execution mode.
-  AMDGPUKernelTy(const char *Name, OMPTgtExecModeFlags ExecutionMode)
+  AMDGPUKernelTy(const char *Name, OMPTgtExecModeFlags ExecutionMode,
+                 uint16_t CodeObjectVersion)
       : GenericKernelTy(Name, ExecutionMode),
-        ImplicitArgsSize(sizeof(utils::AMDGPUImplicitArgsTy)) {}
+        ImplicitArgsSize(utils::getImplicitArgsSize(CodeObjectVersion)) {}
 
   /// Initialize the AMDGPU kernel.
   Error initImpl(GenericDeviceTy &Device, DeviceImageTy &Image) override {
     AMDGPUDeviceImageTy &AMDImage = static_cast<AMDGPUDeviceImageTy &>(Image);
 
     // Kernel symbols have a ".kd" suffix.
     std::string KernelName(getName());
     KernelName += ".kd";
if (auto Err = GHandler.readGlobalFromImage(*this, Image, ExecModeGlobal)) {
       ExecModeGlobal.setValue(llvm::omp::OMP_TGT_EXEC_MODE_GENERIC);
     }
 
     // Check that the retrieved execution mode is valid.
     if (!GenericKernelTy::isValidExecutionMode(ExecModeGlobal.getValue()))
       return Plugin::error("Invalid execution mode %d for '%s'",
                            ExecModeGlobal.getValue(), KernelEntry.name);
 
+    uint16_t Cov = 0;
+    if (auto Err = GHandler.readCOVFromELFHeader(*this, Image, Cov)) {
+      return Plugin::error("Failed to read code object version from the ELF"
+                           "header for the kernel '%s'\n",
+                           KernelEntry.name);
+    }
+
     // Allocate and initialize the AMDGPU kernel.
     AMDGPUKernelTy *AMDKernel = Plugin::get().allocate<AMDGPUKernelTy>();
-    new (AMDKernel) AMDGPUKernelTy(KernelEntry.name, ExecModeGlobal.getValue());
+    new (AMDKernel)
+        AMDGPUKernelTy(KernelEntry.name, ExecModeGlobal.getValue(), Cov);
 
     return AMDKernel;
   }
 
   /// Set the current context to this device's context. Do nothing since the
   /// AMDGPU devices do not have the concept of contexts.
   Error setContext() override { return Plugin::success(); }
 

 AMDGPUStreamTy::AMDGPUStreamTy(AMDGPUDeviceTy &Device)
     : Agent(Device.getAgent()), Queue(Device.getNextQueue()),
       SignalManager(Device.getSignalManager()),
       // Initialize the std::deque with some empty positions.
       Slots(32), NextSlot(0), SyncCycle(0) {}
 
 /// Class implementing the AMDGPU-specific functionalities of the global
 /// handler.
 struct AMDGPUGlobalHandlerTy final : public GenericGlobalHandlerTy {
   /// Get the metadata of a global from the device. The name and size of the
   /// global is read from DeviceGlobal and the address of the global is written
   /// to DeviceGlobal.
   Error getGlobalMetadataFromDevice(GenericDeviceTy &Device,
                                     DeviceImageTy &Image,
                                     GlobalTy &DeviceGlobal) override {
     AMDGPUDeviceImageTy &AMDImage = static_cast<AMDGPUDeviceImageTy &>(Image);
 
Error AMDGPUKernelTy::launchImpl(GenericDeviceTy &GenericDevice,
   AMDHostDeviceTy &HostDevice = Plugin::get<AMDGPUPluginTy>().getHostDevice();
   AMDGPUMemoryManagerTy &ArgsMemoryManager = HostDevice.getArgsMemoryManager();
 
   void *AllArgs = nullptr;
   if (auto Err = ArgsMemoryManager.allocate(AllArgsSize, &AllArgs))
     return Err;
 
   // Initialize implicit arguments.
-  utils::AMDGPUImplicitArgsTy *ImplArgs =
-      reinterpret_cast<utils::AMDGPUImplicitArgsTy *>(
-          static_cast<char *>(AllArgs) + KernelArgsSize);
+  uint8_t *ImplArgs = static_cast<uint8_t *>(AllArgs) + KernelArgsSize;
 
   // Initialize the implicit arguments to zero.
   std::memset(ImplArgs, 0, ImplicitArgsSize);
 
   // Copy the explicit arguments.
   // TODO: We should expose the args memory manager alloc to the common part as
   //     alternative to copying them twice.
   if (NumKernelArgs)

openmp/libomptarget/plugins-nextgen/amdgpu/utils/UtilitiesRTL.h

 #include "llvm/ADT/StringRef.h"
 
 namespace llvm {
 namespace omp {
 namespace target {
 namespace plugin {
 namespace utils {
 
-// The implicit arguments of AMDGPU kernels.
-struct AMDGPUImplicitArgsTy {
-  uint64_t OffsetX;
-  uint64_t OffsetY;
-  uint64_t OffsetZ;
-  uint64_t HostcallPtr;
-  uint64_t Unused0;
-  uint64_t Unused1;
-  uint64_t Unused2;
-};
-
-static_assert(sizeof(AMDGPUImplicitArgsTy) == 56,
-              "Unexpected size of implicit arguments");
+const uint16_t getImplicitArgsSize(uint16_t Version) {
+  return Version < ELF::ELFABIVERSION_AMDGPU_HSA_V5 ? 56 : 256;
+}
 
 /// Parse a TargetID to get processor arch and feature map.
 /// Returns processor subarch.
 /// Returns TargetID features in \p FeatureMap argument.
 /// If the \p TargetID contains feature+, FeatureMap it to true.
 /// If the \p TargetID contains feature-, FeatureMap it to false.
 /// If the \p TargetID does not contain a feature (default), do not map it.
 StringRef parseTargetID(StringRef TargetID, StringMap<bool> &FeatureMap) {

openmp/libomptarget/plugins-nextgen/common/PluginInterface/GlobalHandler.h

public:
   Error getGlobalMetadataFromImage(GenericDeviceTy &Device,
                                    DeviceImageTy &Image, GlobalTy &ImageGlobal);
 
   /// Read the memory associated with a global from the image and store it on
   /// the host. The name, size, and destination are defined by \p HostGlobal.
   Error readGlobalFromImage(GenericDeviceTy &Device, DeviceImageTy &Image,
                             const GlobalTy &HostGlobal);
 
+  Error readCOVFromELFHeader(GenericDeviceTy &Device, DeviceImageTy &Image,
+                             uint16_t &CodeObjectVersion);
+
   /// Get the address and size of a global from the device. Address is return in
   /// \p DeviceGlobal, the global name and expected size are passed in
   /// \p DeviceGlobal.
   virtual Error getGlobalMetadataFromDevice(GenericDeviceTy &Device,
                                             DeviceImageTy &Image,
                                             GlobalTy &DeviceGlobal) = 0;
 
   /// Copy the memory associated with a global from the device to its

openmp/libomptarget/plugins-nextgen/common/PluginInterface/GlobalHandler.cpp

 #include "PluginInterface.h"
 
 #include <cstring>
 
 using namespace llvm;
 using namespace omp;
 using namespace target;
 using namespace plugin;
+using namespace llvm::ELF;
 
 const ELF64LEObjectFile *
 GenericGlobalHandlerTy::getOrCreateELFObjectFile(const GenericDeviceTy &Device,
                                                  DeviceImageTy &Image) {
 
   auto Search = ELFObjectFiles.find(Image.getId());
   if (Search != ELFObjectFiles.end())
     // The ELF object file was already there.
if (!SecOrErr)
     return Plugin::error("Failed to get ELF section from global '%s': %s",
                          ImageGlobal.getName().data(),
                          toString(SecOrErr.takeError()).data());
 
   // Setup the global symbol's address and size.
   return getGlobalMetadataFromELF(Image, **SymOrErr, **SecOrErr, ImageGlobal);
 }
 
+Error GenericGlobalHandlerTy::readCOVFromELFHeader(
+    GenericDeviceTy &Device, DeviceImageTy &Image,
+    uint16_t &CodeObjectVersion) {
+
+  // Get the ELF object file for the image. Notice the ELF object may already
+  // be created in previous calls, so we can reuse it.
+  const ELF64LEObjectFile *ELFObj = getOrCreateELFObjectFile(Device, Image);
+  if (!ELFObj)
+    return Plugin::error("Unable to create ELF object for image %p",
+                         Image.getStart());
+
+  // Get the header and lookup into e_ident.
+  CodeObjectVersion =
+      (uint8_t)ELFObj->getELFFile().getHeader().e_ident[EI_ABIVERSION];
+  if (CodeObjectVersion == 0)
+    return Plugin::error("Failed to get code object version from the ELF"
+                         "header\n");
+
+  return Plugin::success();
+}
+
 Error GenericGlobalHandlerTy::readGlobalFromImage(GenericDeviceTy &Device,
                                                   DeviceImageTy &Image,
                                                   const GlobalTy &HostGlobal) {
 
   GlobalTy ImageGlobal(HostGlobal.getName(), -1);
   if (auto Err = getGlobalMetadataFromImage(Device, Image, ImageGlobal))
     return Err;
 

openmp/libomptarget/plugins/amdgpu/impl/get_elf_mach_gfx_name.h

enum IMPLICITARGS : uint16_t {
   COV5_GRID_DIMS_SIZE = 2,
 
   COV5_HOSTCALL_PTR_OFFSET = 80,
 
   COV5_HEAPV1_PTR_OFFSET = 96,
   COV5_HEAPV1_PTR_SIZE = 8
 };
 
-const uint16_t getImplicitArgsSize(uint16_t Version);
-
 #endif