This is an archive of the discontinued LLVM Phabricator instance.

Differential D154548

[Libomptarget] Remove the remote and ve plugins from libomptarget
ClosedPublic

Authored by jhuber6 on Jul 5 2023, 2:50 PM.

Details

Reviewers
RaviNarayanaswamy
ronlieb
carlo.bertolli
jdoerfert
JonChesterfield
ye-luo
tianshilei1992
atmnpatel
Commits
rG70c08dbcfb8e: [Libomptarget] Remove the remote and ve plugins from libomptarget
Summary

These plugins are unmaintained and are not in a workable state. The VE
plugin has not been touched for years and has never had any running
tests. The remote plugin is in an unfinished state and is not production
ready upstream. These will need to be ported to the new nextgen
interface in the future if they are needed.

Diff Detail

Event Timeline

jhuber6 created this revision.Jul 5 2023, 2:50 PM
Herald added a project: Restricted Project. · View Herald TranscriptJul 5 2023, 2:50 PM
Herald added a subscriber: mgrang. · View Herald Transcript
jhuber6 requested review of this revision.Jul 5 2023, 2:50 PM
Herald added a project: Restricted Project. · View Herald TranscriptJul 5 2023, 2:50 PM
Herald added subscribers: openmp-commits, jplehr, sstefan1. · View Herald Transcript
Harbormaster completed remote builds in B243320: Diff 537512.Jul 5 2023, 2:54 PM
jhuber6 added a child revision: D142820: [OpenMP] Delete old plugins.Jul 5 2023, 2:57 PM
jhuber6 mentioned this in D142820: [OpenMP] Delete old plugins.Jul 5 2023, 2:59 PM
jdoerfert accepted this revision.Jul 5 2023, 3:35 PM

We can always get them back. LG

This revision is now accepted and ready to land.Jul 5 2023, 3:35 PM
Closed by commit rG70c08dbcfb8e: [Libomptarget] Remove the remote and ve plugins from libomptarget (authored by jhuber6). · Explain WhyJul 5 2023, 3:40 PM
This revision was automatically updated to reflect the committed changes.
jhuber6 added a commit: rG70c08dbcfb8e: [Libomptarget] Remove the remote and ve plugins from libomptarget.

Revision Contents

Diff 537529

openmp/libomptarget/cmake/Modules/LibomptargetGetDependencies.cmake

# #
#//===----------------------------------------------------------------------===// #//===----------------------------------------------------------------------===//
#// #//
#// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. #// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
#// See https://llvm.org/LICENSE.txt for license information. #// See https://llvm.org/LICENSE.txt for license information.
#// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#// #//
#//===----------------------------------------------------------------------===// #//===----------------------------------------------------------------------===//
# #
# Try to detect in the system several dependencies required by the different # Try to detect in the system several dependencies required by the different
# components of libomptarget. These are the dependencies we have: # components of libomptarget. These are the dependencies we have:
# #
# libffi : required to launch target kernels given function and argument # libffi : required to launch target kernels given function and argument
# pointers. # pointers.
# CUDA : required to control offloading to NVIDIA GPUs. # CUDA : required to control offloading to NVIDIA GPUs.
# VEOS : required to control offloading to NEC Aurora.
include (FindPackageHandleStandardArgs) include (FindPackageHandleStandardArgs)
################################################################################ ################################################################################
# Looking for LLVM... # Looking for LLVM...
################################################################################ ################################################################################
if (OPENMP_STANDALONE_BUILD) if (OPENMP_STANDALONE_BUILD)
▲ Show 20 Lines • Show All 121 Lines • ▼ Show 20 Lines execute_process(COMMAND ${LIBOMPTARGET_AMDGPU_ARCH}
OUTPUT_STRIP_TRAILING_WHITESPACE) OUTPUT_STRIP_TRAILING_WHITESPACE)
string(REPLACE "\n" ";" amdgpu_arch_list "${LIBOMPTARGET_AMDGPU_ARCH_OUTPUT}") string(REPLACE "\n" ";" amdgpu_arch_list "${LIBOMPTARGET_AMDGPU_ARCH_OUTPUT}")
if(amdgpu_arch_list) if(amdgpu_arch_list)
set(LIBOMPTARGET_FOUND_AMDGPU_GPU TRUE) set(LIBOMPTARGET_FOUND_AMDGPU_GPU TRUE)
set(LIBOMPTARGET_AMDGPU_DETECTED_ARCH_LIST "${amdgpu_arch_list}") set(LIBOMPTARGET_AMDGPU_DETECTED_ARCH_LIST "${amdgpu_arch_list}")
endif() endif()
endif() endif()
################################################################################
# Looking for VEO...
################################################################################
find_path (
LIBOMPTARGET_DEP_VEO_INCLUDE_DIR
NAMES
ve_offload.h
PATHS
/usr/include
/usr/local/include
/opt/local/include
/sw/include
/opt/nec/ve/veos/include
ENV CPATH
PATH_SUFFIXES
libveo)
find_library (
LIBOMPTARGET_DEP_VEO_LIBRARIES
NAMES
veo
PATHS
/usr/lib
/usr/local/lib
/opt/local/lib
/sw/lib
/opt/nec/ve/veos/lib64
ENV LIBRARY_PATH
ENV LD_LIBRARY_PATH)
find_library(
LIBOMPTARGET_DEP_VEOSINFO_LIBRARIES
NAMES
veosinfo
PATHS
/usr/lib
/usr/local/lib
/opt/local/lib
/sw/lib
/opt/nec/ve/veos/lib64
ENV LIBRARY_PATH
ENV LD_LIBRARY_PATH)
set(LIBOMPTARGET_DEP_VEO_INCLUDE_DIRS ${LIBOMPTARGET_DEP_VEO_INCLUDE_DIR})
find_package_handle_standard_args(
LIBOMPTARGET_DEP_VEO
DEFAULT_MSG
LIBOMPTARGET_DEP_VEO_LIBRARIES
LIBOMPTARGET_DEP_VEOSINFO_LIBRARIES
LIBOMPTARGET_DEP_VEO_INCLUDE_DIRS)
mark_as_advanced(
LIBOMPTARGET_DEP_VEO_FOUND
LIBOMPTARGET_DEP_VEO_INCLUDE_DIRS)
set(OPENMP_PTHREAD_LIB ${LLVM_PTHREAD_LIB}) set(OPENMP_PTHREAD_LIB ${LLVM_PTHREAD_LIB})

openmp/libomptarget/plugins/CMakeLists.txt

Show First 20 Lines • Show All 75 Lines • ▼ Show 20 Lines
endif() endif()
endmacro() endmacro()
add_subdirectory(aarch64) add_subdirectory(aarch64)
add_subdirectory(amdgpu) add_subdirectory(amdgpu)
add_subdirectory(cuda) add_subdirectory(cuda)
add_subdirectory(ppc64) add_subdirectory(ppc64)
add_subdirectory(ppc64le) add_subdirectory(ppc64le)
add_subdirectory(ve)
add_subdirectory(x86_64) add_subdirectory(x86_64)
add_subdirectory(remote)
# Make sure the parent scope can see the plugins that will be created. # Make sure the parent scope can see the plugins that will be created.
set(LIBOMPTARGET_SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS}" PARENT_SCOPE) set(LIBOMPTARGET_SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS}" PARENT_SCOPE)
set(LIBOMPTARGET_TESTED_PLUGINS "${LIBOMPTARGET_TESTED_PLUGINS}" PARENT_SCOPE) set(LIBOMPTARGET_TESTED_PLUGINS "${LIBOMPTARGET_TESTED_PLUGINS}" PARENT_SCOPE)

openmp/libomptarget/plugins/remote/CMakeLists.txt

  • This file was deleted.
##===----------------------------------------------------------------------===##
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
##===----------------------------------------------------------------------===##
#
# Build a plugin (client) and server for remote offloading.
#
##===----------------------------------------------------------------------===#
if (NOT(CMAKE_SYSTEM_NAME MATCHES "Linux"))
libomptarget_say("Not building remote offloading plugin: only support Linux hosts.")
return()
endif()
if (NOT(LIBOMPTARGET_ENABLE_EXPERIMENTAL_REMOTE_PLUGIN))
return()
endif()
find_package(Protobuf)
find_package(gRPC CONFIG)
find_program(PROTOC protoc)
find_program(GRPC_CPP_PLUGIN grpc_cpp_plugin)
if (Protobuf_FOUND AND gRPC_FOUND AND PROTOC AND GRPC_CPP_PLUGIN)
libomptarget_say("Building remote offloading plugin.")
set(directory "${CMAKE_BINARY_DIR}/include/openmp/libomptarget/plugins/remote/")
file(MAKE_DIRECTORY ${directory})
execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${directory})
execute_process(
COMMAND protoc --cpp_out=${directory} -I ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/include/openmp.proto
COMMAND protoc --grpc_out=${directory} -I ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/include/openmp.proto --plugin=protoc-gen-grpc=${GRPC_CPP_PLUGIN}
)
set(GRPC_SRC_FILES
${directory}/openmp.grpc.pb.cc
${directory}/openmp.pb.cc
)
set(GRPC_INCLUDE_DIR
${directory}
)
set(RPC_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/include/)
set(RPC_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/lib/)
add_subdirectory(src)
add_subdirectory(server)
else()
libomptarget_say("Not building remote offloading plugin: required libraries were not found.")
endif()

openmp/libomptarget/plugins/remote/include/Utils.h

  • This file was deleted.
//===----------------- Utils.h - Utilities for Remote RTL -----------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// Utilities for data transfer through protobuf and debugging.
//
//===----------------------------------------------------------------------===//
#ifndef UTILS_H
#define UTILS_H
#include "Debug.h"
#include "omptarget.h"
#include "openmp.grpc.pb.h"
#include "openmp.pb.h"
#include "rtl.h"
#include <string>
#define CLIENT_DBG(...) \
{ \
if (DebugLevel > 0) { \
fprintf(stderr, "[[Client]] --> "); \
fprintf(stderr, __VA_ARGS__); \
fprintf(stderr, "\n"); \
} \
}
#define SERVER_DBG(...) \
{ \
if (DebugLevel > 0) { \
fprintf(stderr, "[[Server]] --> "); \
fprintf(stderr, __VA_ARGS__); \
fprintf(stderr, "\n"); \
} \
}
namespace RemoteOffloading {
using namespace openmp::libomptarget::remote;
using openmp::libomptarget::remote::DeviceOffloadEntry;
using openmp::libomptarget::remote::TargetBinaryDescription;
using openmp::libomptarget::remote::TargetOffloadEntry;
using openmp::libomptarget::remote::TargetTable;
struct ClientManagerConfigTy {
std::vector<std::string> ServerAddresses;
uint64_t MaxSize;
uint64_t BlockSize;
int Timeout;
ClientManagerConfigTy()
: ServerAddresses({"0.0.0.0:50051"}), MaxSize(1 << 30),
BlockSize(1 << 20), Timeout(5) {
// TODO: Error handle for incorrect inputs
if (const char *Env = std::getenv("LIBOMPTARGET_RPC_ADDRESS")) {
ServerAddresses.clear();
std::string AddressString = Env;
const std::string Delimiter = ",";
size_t Pos;
std::string Token;
while ((Pos = AddressString.find(Delimiter)) != std::string::npos) {
Token = AddressString.substr(0, Pos);
ServerAddresses.push_back(Token);
AddressString.erase(0, Pos + Delimiter.length());
}
ServerAddresses.push_back(AddressString);
}
if (const char *Env = std::getenv("LIBOMPTARGET_RPC_ALLOCATOR_MAX"))
MaxSize = std::stoi(Env);
if (const char *Env = std::getenv("LIBOMPTARGET_RPC_BLOCK_SIZE"))
BlockSize = std::stoi(Env);
if (const char *Env1 = std::getenv("LIBOMPTARGET_RPC_LATENCY"))
Timeout = std::stoi(Env1);
}
};
/// Loads a target binary description into protobuf.
void loadTargetBinaryDescription(const __tgt_bin_desc *Desc,
TargetBinaryDescription &Request);
/// Unload a target binary description from protobuf. The map is used to keep
/// track of already copied device images.
void unloadTargetBinaryDescription(
const TargetBinaryDescription *Request, __tgt_bin_desc *Desc,
std::unordered_map<const void *, __tgt_device_image *>
&HostToRemoteDeviceImage);
/// Frees argument as constructed by loadTargetBinaryDescription
void freeTargetBinaryDescription(__tgt_bin_desc *Desc);
/// Copies from TargetOffloadEntry protobuf to a tgt_bin_desc during unloading.
void copyOffloadEntry(const TargetOffloadEntry &EntryResponse,
__tgt_offload_entry *Entry);
/// Copies from tgt_bin_desc into TargetOffloadEntry protobuf during loading.
void copyOffloadEntry(const __tgt_offload_entry *Entry,
TargetOffloadEntry *EntryResponse);
/// Shallow copy of offload entry from tgt_bin_desc to TargetOffloadEntry
/// during loading.
void shallowCopyOffloadEntry(const __tgt_offload_entry *Entry,
TargetOffloadEntry *EntryResponse);
/// Copies DeviceOffloadEntries into table during unloading.
void copyOffloadEntry(const DeviceOffloadEntry &EntryResponse,
__tgt_offload_entry *Entry);
/// Loads tgt_target_table into a TargetTable protobuf message.
void loadTargetTable(__tgt_target_table *Table, TargetTable &TableResponse,
__tgt_device_image *Image);
/// Unloads from a target_table from protobuf.
void unloadTargetTable(
TargetTable &TableResponse, __tgt_target_table *Table,
std::unordered_map<void *, void *> &HostToRemoteTargetTableMap);
/// Frees argument as constructed by unloadTargetTable
void freeTargetTable(__tgt_target_table *Table);
void dump(const void *Start, const void *End);
void dump(__tgt_offload_entry *Entry);
void dump(TargetOffloadEntry Entry);
void dump(__tgt_target_table *Table);
void dump(__tgt_device_image *Image);
} // namespace RemoteOffloading
#endif

openmp/libomptarget/plugins/remote/include/openmp.proto

  • This file was deleted.
syntax = "proto3";
package openmp.libomptarget.remote;
option cc_enable_arenas = true;
service RemoteOffload {
rpc Shutdown(Null) returns (I32) {}
rpc RegisterLib(TargetBinaryDescription) returns (I32) {}
rpc UnregisterLib(Pointer) returns (I32) {}
rpc IsValidBinary(TargetDeviceImagePtr) returns (I32) {}
rpc GetNumberOfDevices(Null) returns (I32) {}
rpc InitDevice(I32) returns (I32) {}
rpc InitRequires(I64) returns (I32) {}
rpc LoadBinary(Binary) returns (TargetTable) {}
rpc DataAlloc(AllocData) returns (Pointer) {}
rpc DataDelete(DeleteData) returns (I32) {}
rpc DataSubmit(stream SubmitData) returns (I32) {}
rpc DataRetrieve(RetrieveData) returns (stream Data) {}
rpc IsDataExchangeable(DevicePair) returns (I32) {}
rpc DataExchange(ExchangeData) returns (I32) {}
rpc RunTargetRegion(TargetRegion) returns (I32) {}
rpc RunTargetTeamRegion(TargetTeamRegion) returns (I32) {}
}
message Null {}
message Pointer { uint64 number = 1; }
message I32 { int32 number = 1; }
message I64 { int64 number = 1; }
message DevicePair {
int32 src_dev_id = 1;
int32 dst_dev_id = 2;
}
message Binary {
uint64 image_ptr = 1;
int32 device_id = 2;
}
message TargetOffloadEntry {
bytes data = 1;
string name = 2;
int32 flags = 3;
int32 reserved = 4;
}
message DeviceOffloadEntry {
string name = 1;
uint64 addr = 2;
int32 flags = 3;
int32 reserved = 4;
int32 size = 5;
}
message TargetTable {
repeated DeviceOffloadEntry entries = 1;
repeated uint64 entry_ptrs = 2;
}
message TargetDeviceImagePtr {
uint64 image_ptr = 1;
repeated uint64 entry_ptrs = 2;
}
message TargetDeviceImage {
bytes binary = 1;
repeated TargetOffloadEntry entries = 2;
}
message ImagePtrs {
uint64 img_ptr = 1;
repeated uint64 entry_ptrs = 2;
}
message TargetBinaryDescription {
repeated ImagePtrs image_ptrs = 1;
repeated TargetOffloadEntry entries = 2;
repeated TargetDeviceImage images = 3;
repeated uint64 entry_ptrs = 4;
uint64 bin_ptr = 5;
}
message AllocData {
uint64 size = 1;
uint64 hst_ptr = 2;
int32 device_id = 3;
}
message SubmitData {
bytes data = 1;
uint64 hst_ptr = 2;
uint64 tgt_ptr = 3;
uint64 start = 5;
uint64 size = 6;
int32 device_id = 7;
}
message RetrieveData {
uint64 hst_ptr = 1;
uint64 tgt_ptr = 2;
uint64 size = 3;
int32 device_id = 5;
}
message Data {
bytes data = 1;
uint64 start = 2;
uint64 size = 3;
int32 ret = 4;
}
message ExchangeData {
uint64 src_dev_id = 1;
uint64 src_ptr = 2;
uint64 dst_dev_id = 3;
uint64 dst_ptr = 4;
uint64 size = 6;
}
message DeleteData {
uint64 tgt_ptr = 1;
int32 device_id = 2;
}
message TargetRegion {
repeated uint64 tgt_args = 1;
repeated int64 tgt_offsets = 2;
uint64 tgt_entry_ptr = 3;
int32 device_id = 4;
int32 arg_num = 5;
}
message TargetTeamRegion {
repeated uint64 tgt_args = 1;
repeated int64 tgt_offsets = 2;
uint64 tgt_entry_ptr = 3;
uint64 loop_tripcount = 4;
int32 device_id = 5;
int32 arg_num = 6;
int32 team_num = 7;
int32 thread_limit = 8;
}

openmp/libomptarget/plugins/remote/lib/Utils.cpp

  • This file was deleted.
//===---------------- Utils.cpp - Utilities for Remote RTL ----------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// Utilities for data movement and debugging.
//
//===----------------------------------------------------------------------===//
#include "Utils.h"
#include "omptarget.h"
namespace RemoteOffloading {
void loadTargetBinaryDescription(const __tgt_bin_desc *Desc,
TargetBinaryDescription &Request) {
// Keeps track of entries which have already been deep copied.
std::vector<void *> DeepCopiedEntryAddrs;
// Copy Global Offload Entries
for (auto *CurEntry = Desc->HostEntriesBegin;
CurEntry != Desc->HostEntriesEnd; CurEntry++) {
auto *NewEntry = Request.add_entries();
copyOffloadEntry(CurEntry, NewEntry);
// Copy the pointer of the offload entry of the image into the Request
Request.add_entry_ptrs((uint64_t)CurEntry);
DeepCopiedEntryAddrs.push_back(CurEntry);
}
// Copy Device Images and Device Offload Entries
__tgt_device_image *CurImage = Desc->DeviceImages;
for (auto I = 0; I < Desc->NumDeviceImages; I++, CurImage++) {
auto *Image = Request.add_images();
auto Size = (char *)CurImage->ImageEnd - (char *)CurImage->ImageStart;
Image->set_binary(CurImage->ImageStart, Size);
// Copy the pointer of the image into the Request
auto *NewImagePtr = Request.add_image_ptrs();
NewImagePtr->set_img_ptr((uint64_t)CurImage->ImageStart);
// Copy Device Offload Entries
for (auto *CurEntry = CurImage->EntriesBegin;
CurEntry != CurImage->EntriesEnd; CurEntry++) {
auto *NewEntry = Image->add_entries();
auto Entry = std::find(DeepCopiedEntryAddrs.begin(),
DeepCopiedEntryAddrs.end(), CurEntry);
if (Entry != DeepCopiedEntryAddrs.end()) {
// Offload entry has already been loaded
shallowCopyOffloadEntry(CurEntry, NewEntry);
} else { // Offload Entry has not been loaded into the Request
copyOffloadEntry(CurEntry, NewEntry);
DeepCopiedEntryAddrs.push_back(CurEntry);
}
// Copy the pointer of the offload entry of the image into the Request
NewImagePtr->add_entry_ptrs((uint64_t)CurEntry);
}
}
}
void unloadTargetBinaryDescription(
const TargetBinaryDescription *Request, __tgt_bin_desc *Desc,
std::unordered_map<const void *, __tgt_device_image *>
&HostToRemoteDeviceImage) {
std::unordered_map<const void *, __tgt_offload_entry *> CopiedOffloadEntries;
Desc->NumDeviceImages = Request->images_size();
Desc->DeviceImages = new __tgt_device_image[Desc->NumDeviceImages];
if (Request->entries_size())
Desc->HostEntriesBegin = new __tgt_offload_entry[Request->entries_size()];
else {
Desc->HostEntriesBegin = nullptr;
Desc->HostEntriesEnd = nullptr;
}
// Copy Global Offload Entries
__tgt_offload_entry *CurEntry = Desc->HostEntriesBegin;
size_t I = 0;
for (auto &Entry : Request->entries()) {
copyOffloadEntry(Entry, CurEntry);
CopiedOffloadEntries[(void *)Request->entry_ptrs()[I]] = CurEntry;
CurEntry++;
I++;
}
Desc->HostEntriesEnd = CurEntry;
// Copy Device Images and Device Offload Entries
__tgt_device_image *CurImage = Desc->DeviceImages;
auto ImageItr = Request->image_ptrs().begin();
for (auto Image : Request->images()) {
// Copy Device Offload Entries
CurEntry = Desc->HostEntriesBegin;
bool Found = false;
if (!Desc->HostEntriesBegin) {
CurImage->EntriesBegin = nullptr;
CurImage->EntriesEnd = nullptr;
}
for (size_t I = 0; I < Image.entries_size(); I++) {
auto TgtEntry =
CopiedOffloadEntries.find((void *)Request->entry_ptrs()[I]);
if (TgtEntry != CopiedOffloadEntries.end()) {
if (!Found)
CurImage->EntriesBegin = CurEntry;
CurImage->EntriesEnd = CurEntry + 1;
Found = true;
} else {
Found = false;
copyOffloadEntry(Image.entries()[I], CurEntry);
CopiedOffloadEntries[(void *)(Request->entry_ptrs()[I])] = CurEntry;
}
CurEntry++;
}
// Copy Device Image
CurImage->ImageStart = new uint8_t[Image.binary().size()];
memcpy(CurImage->ImageStart,
static_cast<const void *>(Image.binary().data()),
Image.binary().size());
CurImage->ImageEnd =
(void *)((char *)CurImage->ImageStart + Image.binary().size());
HostToRemoteDeviceImage[(void *)ImageItr->img_ptr()] = CurImage;
CurImage++;
ImageItr++;
}
}
void freeTargetBinaryDescription(__tgt_bin_desc *Desc) {
__tgt_device_image *CurImage = Desc->DeviceImages;
for (auto I = 0; I < Desc->NumDeviceImages; I++, CurImage++)
delete[](uint64_t *) CurImage->ImageStart;
delete[] Desc->DeviceImages;
for (auto *Entry = Desc->HostEntriesBegin; Entry != Desc->HostEntriesEnd;
Entry++) {
free(Entry->name);
free(Entry->addr);
}
delete[] Desc->HostEntriesBegin;
}
void freeTargetTable(__tgt_target_table *Table) {
for (auto *Entry = Table->EntriesBegin; Entry != Table->EntriesEnd; Entry++)
free(Entry->name);
delete[] Table->EntriesBegin;
}
void loadTargetTable(__tgt_target_table *Table, TargetTable &TableResponse,
__tgt_device_image *Image) {
auto *ImageEntry = Image->EntriesBegin;
for (__tgt_offload_entry *CurEntry = Table->EntriesBegin;
CurEntry != Table->EntriesEnd; CurEntry++, ImageEntry++) {
// TODO: This can probably be trimmed substantially.
auto *NewEntry = TableResponse.add_entries();
NewEntry->set_name(CurEntry->name);
NewEntry->set_addr((uint64_t)CurEntry->addr);
NewEntry->set_flags(CurEntry->flags);
NewEntry->set_reserved(CurEntry->reserved);
NewEntry->set_size(CurEntry->size);
TableResponse.add_entry_ptrs((int64_t)CurEntry);
}
}
void unloadTargetTable(
TargetTable &TableResponse, __tgt_target_table *Table,
std::unordered_map<void *, void *> &HostToRemoteTargetTableMap) {
Table->EntriesBegin = new __tgt_offload_entry[TableResponse.entries_size()];
auto *CurEntry = Table->EntriesBegin;
for (size_t I = 0; I < TableResponse.entries_size(); I++) {
copyOffloadEntry(TableResponse.entries()[I], CurEntry);
HostToRemoteTargetTableMap[CurEntry->addr] =
(void *)TableResponse.entry_ptrs()[I];
CurEntry++;
}
Table->EntriesEnd = CurEntry;
}
void copyOffloadEntry(const TargetOffloadEntry &EntryResponse,
__tgt_offload_entry *Entry) {
Entry->name = strdup(EntryResponse.name().c_str());
Entry->reserved = EntryResponse.reserved();
Entry->flags = EntryResponse.flags();
Entry->addr = strdup(EntryResponse.data().c_str());
Entry->size = EntryResponse.data().size();
}
void copyOffloadEntry(const DeviceOffloadEntry &EntryResponse,
__tgt_offload_entry *Entry) {
Entry->name = strdup(EntryResponse.name().c_str());
Entry->reserved = EntryResponse.reserved();
Entry->flags = EntryResponse.flags();
Entry->addr = (void *)EntryResponse.addr();
Entry->size = EntryResponse.size();
}
/// We shallow copy with just the name because it is a convenient identifier, we
/// do actually just match off of the address.
void shallowCopyOffloadEntry(const __tgt_offload_entry *Entry,
TargetOffloadEntry *EntryResponse) {
EntryResponse->set_name(Entry->name);
}
void copyOffloadEntry(const __tgt_offload_entry *Entry,
TargetOffloadEntry *EntryResponse) {
shallowCopyOffloadEntry(Entry, EntryResponse);
EntryResponse->set_reserved(Entry->reserved);
EntryResponse->set_flags(Entry->flags);
EntryResponse->set_data(Entry->addr, Entry->size);
}
/// Dumps the memory region from Start to End in order to debug memory transfer
/// errors within the plugin
void dump(const void *Start, const void *End) {
unsigned char Line[17];
const unsigned char *PrintCharacter = (const unsigned char *)Start;
unsigned int I = 0;
for (; I < ((const int *)End - (const int *)Start); I++) {
if ((I % 16) == 0) {
if (I != 0)
printf(" %s\n", Line);
printf(" %04x ", I);
}
printf(" %02x", PrintCharacter[I]);
if ((PrintCharacter[I] < 0x20) || (PrintCharacter[I] > 0x7e))
Line[I % 16] = '.';
else
Line[I % 16] = PrintCharacter[I];
Line[(I % 16) + 1] = '\0';
}
while ((I % 16) != 0) {
printf(" ");
I++;
}
printf(" %s\n", Line);
}
void dump(__tgt_offload_entry *Entry) {
fprintf(stderr, "Entry (%p):\n", (void *)Entry);
fprintf(stderr, " Name: %s (%p)\n", Entry->name, (void *)&Entry->name);
fprintf(stderr, " Reserved: %d (%p)\n", Entry->reserved,
(void *)&Entry->reserved);
fprintf(stderr, " Flags: %d (%p)\n", Entry->flags, (void *)&Entry->flags);
fprintf(stderr, " Addr: %p\n", Entry->addr);
fprintf(stderr, " Size: %lu\n", Entry->size);
}
void dump(__tgt_target_table *Table) {
for (auto *CurEntry = Table->EntriesBegin; CurEntry != Table->EntriesEnd;
CurEntry++)
dump(CurEntry);
}
void dump(TargetOffloadEntry Entry) {
fprintf(stderr, "Entry: ");
fprintf(stderr, " Name: %s\n", Entry.name().c_str());
fprintf(stderr, " Reserved: %d\n", Entry.reserved());
fprintf(stderr, " Flags: %d\n", Entry.flags());
fprintf(stderr, " Size: %ld\n", Entry.data().size());
dump(static_cast<const void *>(Entry.data().data()),
static_cast<const void *>((Entry.data().c_str() + Entry.data().size())));
}
void dump(__tgt_device_image *Image) {
dump(Image->ImageStart, Image->ImageEnd);
__tgt_offload_entry *EntryItr = Image->EntriesBegin;
for (; EntryItr != Image->EntriesEnd; EntryItr++)
dump(EntryItr);
}
void dump(std::unordered_map<void *, __tgt_offload_entry *> &Map) {
fprintf(stderr, "Host to Remote Entry Map:\n");
for (auto Entry : Map)
fprintf(stderr, " Host (%p) -> Tgt (%p): Addr((%p))\n", Entry.first,
(void *)Entry.second, (void *)Entry.second->addr);
}
} // namespace RemoteOffloading

openmp/libomptarget/plugins/remote/server/CMakeLists.txt

  • This file was deleted.
##===----------------------------------------------------------------------===##
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
##===----------------------------------------------------------------------===##
#
# Build server for remote offloading.
#
##===----------------------------------------------------------------------===##
include_directories(${LIBOMPTARGET_SRC_DIR})
include_directories(${LIBOMPTARGET_INCLUDE_DIR})
include_directories(${GRPC_INCLUDE_DIR})
include_directories(${RPC_INCLUDE_DIR})
add_executable(openmp-offloading-server
${LIBOMPTARGET_SRC_FILES}
${GRPC_SRC_FILES}
${RPC_SRC_DIR}/Utils.cpp
Server.cpp
OffloadingServer.cpp
)
target_link_libraries(openmp-offloading-server
grpc++
protobuf
absl::synchronization
${OPENMP_PTHREAD_LIB}
omp
"-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../../exports")

openmp/libomptarget/plugins/remote/server/OffloadingServer.cpp

  • This file was deleted.
//===------------- OffloadingServer.cpp - Server Application --------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// Offloading server for remote host.
//
//===----------------------------------------------------------------------===//
#include <future>
#include <grpcpp/server.h>
#include <grpcpp/server_builder.h>
#include <iostream>
#include <thread>
#include "Server.h"
using grpc::Server;
using grpc::ServerBuilder;
std::promise<void> ShutdownPromise;
int main() {
ClientManagerConfigTy Config;
RemoteOffloadImpl Service(Config.MaxSize, Config.BlockSize);
ServerBuilder Builder;
Builder.AddListeningPort(Config.ServerAddresses[0],
grpc::InsecureServerCredentials());
Builder.RegisterService(&Service);
Builder.SetMaxMessageSize(INT_MAX);
std::unique_ptr<Server> Server(Builder.BuildAndStart());
if (getDebugLevel())
std::cerr << "Server listening on " << Config.ServerAddresses[0]
<< std::endl;
auto WaitForServer = [&]() { Server->Wait(); };
std::thread ServerThread(WaitForServer);
auto ShutdownFuture = ShutdownPromise.get_future();
ShutdownFuture.wait();
Server->Shutdown();
ServerThread.join();
return 0;
}

openmp/libomptarget/plugins/remote/server/Server.h

  • This file was deleted.
//===-------------------------- Server.h - Server -------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// Offloading gRPC server for remote host.
//
//===----------------------------------------------------------------------===//
#ifndef LLVM_OPENMP_LIBOMPTARGET_PLUGINS_REMOTE_SERVER_SERVER_H
#define LLVM_OPENMP_LIBOMPTARGET_PLUGINS_REMOTE_SERVER_SERVER_H
#include <grpcpp/server_context.h>
#include "Utils.h"
#include "device.h"
#include "omptarget.h"
#include "openmp.grpc.pb.h"
#include "openmp.pb.h"
#include "rtl.h"
using grpc::ServerContext;
using grpc::ServerReader;
using grpc::ServerWriter;
using grpc::Status;
using namespace openmp::libomptarget::remote;
using namespace RemoteOffloading;
using namespace google;
extern PluginManager *PM;
class RemoteOffloadImpl final : public RemoteOffload::Service {
private:
int32_t mapHostRTLDeviceId(int32_t RTLDeviceID);
std::unordered_map<const void *, __tgt_device_image *>
HostToRemoteDeviceImage;
std::unordered_map<const void *, std::unique_ptr<__tgt_bin_desc>>
Descriptions;
__tgt_target_table *Table = nullptr;
int DebugLevel;
uint64_t MaxSize;
uint64_t BlockSize;
std::unique_ptr<protobuf::Arena> Arena;
public:
RemoteOffloadImpl(uint64_t MaxSize, uint64_t BlockSize)
: MaxSize(MaxSize), BlockSize(BlockSize) {
DebugLevel = getDebugLevel();
Arena = std::make_unique<protobuf::Arena>();
}
Status Shutdown(ServerContext *Context, const Null *Request,
I32 *Reply) override;
Status RegisterLib(ServerContext *Context,
const TargetBinaryDescription *Description,
I32 *Reply) override;
Status UnregisterLib(ServerContext *Context, const Pointer *Request,
I32 *Reply) override;
Status IsValidBinary(ServerContext *Context,
const TargetDeviceImagePtr *Image,
I32 *IsValid) override;
Status GetNumberOfDevices(ServerContext *Context, const Null *Null,
I32 *NumberOfDevices) override;
Status InitDevice(ServerContext *Context, const I32 *DeviceNum,
I32 *Reply) override;
Status InitRequires(ServerContext *Context, const I64 *RequiresFlag,
I32 *Reply) override;
Status LoadBinary(ServerContext *Context, const Binary *Binary,
TargetTable *Reply) override;
Status IsDataExchangeable(ServerContext *Context, const DevicePair *Request,
I32 *Reply) override;
Status DataAlloc(ServerContext *Context, const AllocData *Request,
Pointer *Reply) override;
Status DataSubmit(ServerContext *Context, ServerReader<SubmitData> *Reader,
I32 *Reply) override;
Status DataRetrieve(ServerContext *Context, const RetrieveData *Request,
ServerWriter<Data> *Writer) override;
Status DataExchange(ServerContext *Context, const ExchangeData *Request,
I32 *Reply) override;
Status DataDelete(ServerContext *Context, const DeleteData *Request,
I32 *Reply) override;
Status RunTargetRegion(ServerContext *Context, const TargetRegion *Request,
I32 *Reply) override;
Status RunTargetTeamRegion(ServerContext *Context,
const TargetTeamRegion *Request,
I32 *Reply) override;
};
#endif

openmp/libomptarget/plugins/remote/server/Server.cpp

  • This file was deleted.
//===----------------- Server.cpp - Server Implementation -----------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// Offloading gRPC server for remote host.
//
//===----------------------------------------------------------------------===//
#include <cmath>
#include <future>
#include "Server.h"
#include "omptarget.h"
#include "openmp.grpc.pb.h"
#include "openmp.pb.h"
using grpc::WriteOptions;
extern std::promise<void> ShutdownPromise;
Status RemoteOffloadImpl::Shutdown(ServerContext *Context, const Null *Request,
I32 *Reply) {
SERVER_DBG("Shutting down the server")
Reply->set_number(0);
ShutdownPromise.set_value();
return Status::OK;
}
Status
RemoteOffloadImpl::RegisterLib(ServerContext *Context,
const TargetBinaryDescription *Description,
I32 *Reply) {
auto Desc = std::make_unique<__tgt_bin_desc>();
unloadTargetBinaryDescription(Description, Desc.get(),
HostToRemoteDeviceImage);
PM->RTLs.RegisterLib(Desc.get());
if (Descriptions.find((void *)Description->bin_ptr()) != Descriptions.end())
freeTargetBinaryDescription(
Descriptions[(void *)Description->bin_ptr()].get());
else
Descriptions[(void *)Description->bin_ptr()] = std::move(Desc);
SERVER_DBG("Registered library")
Reply->set_number(0);
return Status::OK;
}
Status RemoteOffloadImpl::UnregisterLib(ServerContext *Context,
const Pointer *Request, I32 *Reply) {
if (Descriptions.find((void *)Request->number()) == Descriptions.end()) {
Reply->set_number(1);
return Status::OK;
}
PM->RTLs.UnregisterLib(Descriptions[(void *)Request->number()].get());
freeTargetBinaryDescription(Descriptions[(void *)Request->number()].get());
Descriptions.erase((void *)Request->number());
SERVER_DBG("Unregistered library")
Reply->set_number(0);
return Status::OK;
}
Status RemoteOffloadImpl::IsValidBinary(ServerContext *Context,
const TargetDeviceImagePtr *DeviceImage,
I32 *IsValid) {
__tgt_device_image *Image =
HostToRemoteDeviceImage[(void *)DeviceImage->image_ptr()];
IsValid->set_number(0);
for (auto &RTL : PM->RTLs.AllRTLs)
if (auto Ret = RTL.is_valid_binary(Image)) {
IsValid->set_number(Ret);
break;
}
SERVER_DBG("Checked if binary (%p) is valid",
(void *)(DeviceImage->image_ptr()))
return Status::OK;
}
Status RemoteOffloadImpl::GetNumberOfDevices(ServerContext *Context,
const Null *Null,
I32 *NumberOfDevices) {
int32_t Devices = 0;
PM->RTLsMtx.lock();
for (auto &RTL : PM->RTLs.AllRTLs)
Devices += RTL.NumberOfDevices;
PM->RTLsMtx.unlock();
NumberOfDevices->set_number(Devices);
SERVER_DBG("Got number of devices")
return Status::OK;
}
Status RemoteOffloadImpl::InitDevice(ServerContext *Context,
const I32 *DeviceNum, I32 *Reply) {
Reply->set_number(PM->Devices[DeviceNum->number()]->RTL->init_device(
mapHostRTLDeviceId(DeviceNum->number())));
SERVER_DBG("Initialized device %d", DeviceNum->number())
return Status::OK;
}
Status RemoteOffloadImpl::InitRequires(ServerContext *Context,
const I64 *RequiresFlag, I32 *Reply) {
for (auto &Device : PM->Devices)
if (Device->RTL->init_requires)
Device->RTL->init_requires(RequiresFlag->number());
Reply->set_number(RequiresFlag->number());
SERVER_DBG("Initialized requires for devices")
return Status::OK;
}
Status RemoteOffloadImpl::LoadBinary(ServerContext *Context,
const Binary *Binary, TargetTable *Reply) {
__tgt_device_image *Image =
HostToRemoteDeviceImage[(void *)Binary->image_ptr()];
Table = PM->Devices[Binary->device_id()]->RTL->load_binary(
mapHostRTLDeviceId(Binary->device_id()), Image);
if (Table)
loadTargetTable(Table, *Reply, Image);
SERVER_DBG("Loaded binary (%p) to device %d", (void *)Binary->image_ptr(),
Binary->device_id())
return Status::OK;
}
Status RemoteOffloadImpl::IsDataExchangeable(ServerContext *Context,
const DevicePair *Request,
I32 *Reply) {
Reply->set_number(-1);
if (PM->Devices[mapHostRTLDeviceId(Request->src_dev_id())]
->RTL->is_data_exchangable)
Reply->set_number(PM->Devices[mapHostRTLDeviceId(Request->src_dev_id())]
->RTL->is_data_exchangable(Request->src_dev_id(),
Request->dst_dev_id()));
SERVER_DBG("Checked if data exchangeable between device %d and device %d",
Request->src_dev_id(), Request->dst_dev_id())
return Status::OK;
}
Status RemoteOffloadImpl::DataAlloc(ServerContext *Context,
const AllocData *Request, Pointer *Reply) {
uint64_t TgtPtr =
(uint64_t)PM->Devices[Request->device_id()]->RTL->data_alloc(
mapHostRTLDeviceId(Request->device_id()), Request->size(),
(void *)Request->hst_ptr(), TARGET_ALLOC_DEFAULT);
Reply->set_number(TgtPtr);
SERVER_DBG("Allocated at " DPxMOD "", DPxPTR((void *)TgtPtr))
return Status::OK;
}
Status RemoteOffloadImpl::DataSubmit(ServerContext *Context,
ServerReader<SubmitData> *Reader,
I32 *Reply) {
SubmitData Request;
uint8_t *HostCopy = nullptr;
while (Reader->Read(&Request)) {
if (Request.start() == 0 && Request.size() == Request.data().size()) {
Reader->SendInitialMetadata();
Reply->set_number(PM->Devices[Request.device_id()]->RTL->data_submit(
mapHostRTLDeviceId(Request.device_id()), (void *)Request.tgt_ptr(),
(void *)Request.data().data(), Request.data().size()));
SERVER_DBG("Submitted %lu bytes async to (%p) on device %d",
Request.data().size(), (void *)Request.tgt_ptr(),
Request.device_id())
return Status::OK;
}
if (!HostCopy) {
HostCopy = new uint8_t[Request.size()];
Reader->SendInitialMetadata();
}
memcpy((void *)((char *)HostCopy + Request.start()), Request.data().data(),
Request.data().size());
}
Reply->set_number(PM->Devices[Request.device_id()]->RTL->data_submit(
mapHostRTLDeviceId(Request.device_id()), (void *)Request.tgt_ptr(),
HostCopy, Request.size()));
delete[] HostCopy;
SERVER_DBG("Submitted %lu bytes to (%p) on device %d", Request.data().size(),
(void *)Request.tgt_ptr(), Request.device_id())
return Status::OK;
}
Status RemoteOffloadImpl::DataRetrieve(ServerContext *Context,
const RetrieveData *Request,
ServerWriter<Data> *Writer) {
auto HstPtr = std::make_unique<char[]>(Request->size());
auto Ret = PM->Devices[Request->device_id()]->RTL->data_retrieve(
mapHostRTLDeviceId(Request->device_id()), HstPtr.get(),
(void *)Request->tgt_ptr(), Request->size());
if (Arena->SpaceAllocated() >= MaxSize)
Arena->Reset();
if (Request->size() > BlockSize) {
uint64_t Start = 0, End = BlockSize;
for (auto I = 0; I < ceil((float)Request->size() / BlockSize); I++) {
auto *Reply = protobuf::Arena::CreateMessage<Data>(Arena.get());
Reply->set_start(Start);
Reply->set_size(Request->size());
Reply->set_data((char *)HstPtr.get() + Start, End - Start);
Reply->set_ret(Ret);
if (!Writer->Write(*Reply)) {
CLIENT_DBG("Broken stream when submitting data")
}
SERVER_DBG("Retrieved %lu-%lu/%lu bytes from (%p) on device %d", Start,
End, Request->size(), (void *)Request->tgt_ptr(),
mapHostRTLDeviceId(Request->device_id()))
Start += BlockSize;
End += BlockSize;
if (End >= Request->size())
End = Request->size();
}
} else {
auto *Reply = protobuf::Arena::CreateMessage<Data>(Arena.get());
Reply->set_start(0);
Reply->set_size(Request->size());
Reply->set_data((char *)HstPtr.get(), Request->size());
Reply->set_ret(Ret);
SERVER_DBG("Retrieved %lu bytes from (%p) on device %d", Request->size(),
(void *)Request->tgt_ptr(),
mapHostRTLDeviceId(Request->device_id()))
Writer->WriteLast(*Reply, WriteOptions());
}
return Status::OK;
}
Status RemoteOffloadImpl::DataExchange(ServerContext *Context,
const ExchangeData *Request,
I32 *Reply) {
if (PM->Devices[Request->src_dev_id()]->RTL->data_exchange) {
int32_t Ret = PM->Devices[Request->src_dev_id()]->RTL->data_exchange(
mapHostRTLDeviceId(Request->src_dev_id()), (void *)Request->src_ptr(),
mapHostRTLDeviceId(Request->dst_dev_id()), (void *)Request->dst_ptr(),
Request->size());
Reply->set_number(Ret);
} else
Reply->set_number(-1);
SERVER_DBG(
"Exchanged data asynchronously from device %d (%p) to device %d (%p) of "
"size %lu",
mapHostRTLDeviceId(Request->src_dev_id()), (void *)Request->src_ptr(),
mapHostRTLDeviceId(Request->dst_dev_id()), (void *)Request->dst_ptr(),
Request->size())
return Status::OK;
}
Status RemoteOffloadImpl::DataDelete(ServerContext *Context,
const DeleteData *Request, I32 *Reply) {
auto Ret = PM->Devices[Request->device_id()]->RTL->data_delete(
mapHostRTLDeviceId(Request->device_id()), (void *)Request->tgt_ptr());
Reply->set_number(Ret);
SERVER_DBG("Deleted data from (%p) on device %d", (void *)Request->tgt_ptr(),
mapHostRTLDeviceId(Request->device_id()))
return Status::OK;
}
Status RemoteOffloadImpl::RunTargetRegion(ServerContext *Context,
const TargetRegion *Request,
I32 *Reply) {
std::vector<uint8_t> TgtArgs(Request->arg_num());
for (auto I = 0; I < Request->arg_num(); I++)
TgtArgs[I] = (uint64_t)Request->tgt_args()[I];
std::vector<ptrdiff_t> TgtOffsets(Request->arg_num());
const auto *TgtOffsetItr = Request->tgt_offsets().begin();
for (auto I = 0; I < Request->arg_num(); I++, TgtOffsetItr++)
TgtOffsets[I] = (ptrdiff_t)*TgtOffsetItr;
void *TgtEntryPtr = ((__tgt_offload_entry *)Request->tgt_entry_ptr())->addr;
int32_t Ret = PM->Devices[Request->device_id()]->RTL->run_region(
mapHostRTLDeviceId(Request->device_id()), TgtEntryPtr,
(void **)TgtArgs.data(), TgtOffsets.data(), Request->arg_num());
Reply->set_number(Ret);
SERVER_DBG("Ran TargetRegion on device %d with %d args",
mapHostRTLDeviceId(Request->device_id()), Request->arg_num())
return Status::OK;
}
Status RemoteOffloadImpl::RunTargetTeamRegion(ServerContext *Context,
const TargetTeamRegion *Request,
I32 *Reply) {
std::vector<uint64_t> TgtArgs(Request->arg_num());
for (auto I = 0; I < Request->arg_num(); I++)
TgtArgs[I] = (uint64_t)Request->tgt_args()[I];
std::vector<ptrdiff_t> TgtOffsets(Request->arg_num());
const auto *TgtOffsetItr = Request->tgt_offsets().begin();
for (auto I = 0; I < Request->arg_num(); I++, TgtOffsetItr++)
TgtOffsets[I] = (ptrdiff_t)*TgtOffsetItr;
void *TgtEntryPtr = ((__tgt_offload_entry *)Request->tgt_entry_ptr())->addr;
int32_t Ret = PM->Devices[Request->device_id()]->RTL->run_team_region(
mapHostRTLDeviceId(Request->device_id()), TgtEntryPtr,
(void **)TgtArgs.data(), TgtOffsets.data(), Request->arg_num(),
Request->team_num(), Request->thread_limit(), Request->loop_tripcount());
Reply->set_number(Ret);
SERVER_DBG("Ran TargetTeamRegion on device %d with %d args",
mapHostRTLDeviceId(Request->device_id()), Request->arg_num())
return Status::OK;
}
int32_t RemoteOffloadImpl::mapHostRTLDeviceId(int32_t RTLDeviceID) {
for (auto &RTL : PM->RTLs.UsedRTLs) {
if (RTLDeviceID - RTL->NumberOfDevices >= 0)
RTLDeviceID -= RTL->NumberOfDevices;
else
break;
}
return RTLDeviceID;
}

openmp/libomptarget/plugins/remote/src/CMakeLists.txt

  • This file was deleted.
##===----------------------------------------------------------------------===##
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
##===----------------------------------------------------------------------===##
#
# Build a plugin for remote offloading.
#
##===----------------------------------------------------------------------===##
cmake_minimum_required(VERSION 3.20.0)
# Define the suffix for the runtime messaging dumps.
add_definitions(-DTARGET_NAME=RPC)
include_directories(${LIBOMPTARGET_SRC_DIR})
include_directories(${LIBOMPTARGET_INCLUDE_DIR})
include_directories(${GRPC_INCLUDE_DIR})
include_directories(${RPC_INCLUDE_DIR})
add_library(omptarget.rtl.rpc SHARED
${LIBOMPTARGET_SRC_FILES}
${GRPC_SRC_FILES}
${RPC_SRC_DIR}/Utils.cpp
Client.cpp
rtl.cpp
)
# Install plugin under the lib destination folder.
install(TARGETS omptarget.rtl.rpc LIBRARY DESTINATION "${OPENMP_INSTALL_LIBDIR}")
target_link_libraries(omptarget.rtl.rpc
grpc++
protobuf
absl::synchronization
${OPENMP_PTHREAD_LIB}
omp
"-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../../exports")
# Report to the parent scope that we are building a plugin for RPC.
set(LIBOMPTARGET_SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS} rpc" PARENT_SCOPE)

openmp/libomptarget/plugins/remote/src/Client.h

  • This file was deleted.
//===------------------ Client.h - Client Implementation ------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// gRPC Client for the remote plugin.
//
//===----------------------------------------------------------------------===//
#ifndef LLVM_OPENMP_LIBOMPTARGET_PLUGINS_REMOTE_SRC_CLIENT_H
#define LLVM_OPENMP_LIBOMPTARGET_PLUGINS_REMOTE_SRC_CLIENT_H
#include "Utils.h"
#include "omptarget.h"
#include <google/protobuf/arena.h>
#include <grpcpp/grpcpp.h>
#include <grpcpp/security/credentials.h>
#include <grpcpp/support/channel_arguments.h>
#include <memory>
#include <mutex>
#include <numeric>
using grpc::Channel;
using openmp::libomptarget::remote::RemoteOffload;
using namespace RemoteOffloading;
using namespace google;
class RemoteOffloadClient {
int DebugLevel;
const int Timeout;
const uint64_t MaxSize;
const int64_t BlockSize;
std::unique_ptr<RemoteOffload::Stub> Stub;
std::unique_ptr<protobuf::Arena> Arena;
std::unique_ptr<std::mutex> ArenaAllocatorLock;
std::map<int32_t, std::unordered_map<void *, void *>> RemoteEntries;
std::map<int32_t, std::unique_ptr<__tgt_target_table>> DevicesToTables;
template <typename Fn1, typename Fn2, typename TReturn>
auto remoteCall(Fn1 Preprocessor, Fn2 Postprocessor, TReturn ErrorValue,
bool CanTimeOut = true);
public:
RemoteOffloadClient(std::shared_ptr<Channel> Channel, int Timeout,
uint64_t MaxSize, int64_t BlockSize)
: Timeout(Timeout), MaxSize(MaxSize), BlockSize(BlockSize),
Stub(RemoteOffload::NewStub(Channel)) {
DebugLevel = getDebugLevel();
Arena = std::make_unique<protobuf::Arena>();
ArenaAllocatorLock = std::make_unique<std::mutex>();
}
RemoteOffloadClient(RemoteOffloadClient &&C) = default;
~RemoteOffloadClient() {
for (auto &TableIt : DevicesToTables)
freeTargetTable(TableIt.second.get());
}
int32_t shutdown(void);
int32_t registerLib(__tgt_bin_desc *Desc);
int32_t unregisterLib(__tgt_bin_desc *Desc);
int32_t isValidBinary(__tgt_device_image *Image);
int32_t getNumberOfDevices();
int32_t initDevice(int32_t DeviceId);
int32_t initRequires(int64_t RequiresFlags);
__tgt_target_table *loadBinary(int32_t DeviceId, __tgt_device_image *Image);
void *dataAlloc(int32_t DeviceId, int64_t Size, void *HstPtr);
int32_t dataDelete(int32_t DeviceId, void *TgtPtr);
int32_t dataSubmit(int32_t DeviceId, void *TgtPtr, void *HstPtr,
int64_t Size);
int32_t dataRetrieve(int32_t DeviceId, void *HstPtr, void *TgtPtr,
int64_t Size);
int32_t isDataExchangeable(int32_t SrcDevId, int32_t DstDevId);
int32_t dataExchange(int32_t SrcDevId, void *SrcPtr, int32_t DstDevId,
void *DstPtr, int64_t Size);
int32_t runTargetRegion(int32_t DeviceId, void *TgtEntryPtr, void **TgtArgs,
ptrdiff_t *TgtOffsets, int32_t ArgNum);
int32_t runTargetTeamRegion(int32_t DeviceId, void *TgtEntryPtr,
void **TgtArgs, ptrdiff_t *TgtOffsets,
int32_t ArgNum, int32_t TeamNum,
int32_t ThreadLimit, uint64_t LoopTripCount);
};
class RemoteClientManager {
private:
std::vector<RemoteOffloadClient> Clients;
std::vector<int> Devices;
std::pair<int32_t, int32_t> mapDeviceId(int32_t DeviceId);
int DebugLevel;
public:
RemoteClientManager() {
ClientManagerConfigTy Config;
grpc::ChannelArguments ChArgs;
ChArgs.SetMaxReceiveMessageSize(-1);
DebugLevel = getDebugLevel();
for (auto Address : Config.ServerAddresses) {
Clients.push_back(RemoteOffloadClient(
grpc::CreateChannel(Address, grpc::InsecureChannelCredentials()),
Config.Timeout, Config.MaxSize, Config.BlockSize));
}
}
int32_t shutdown(void);
int32_t registerLib(__tgt_bin_desc *Desc);
int32_t unregisterLib(__tgt_bin_desc *Desc);
int32_t isValidBinary(__tgt_device_image *Image);
int32_t getNumberOfDevices();
int32_t initDevice(int32_t DeviceId);
int32_t initRequires(int64_t RequiresFlags);
__tgt_target_table *loadBinary(int32_t DeviceId, __tgt_device_image *Image);
void *dataAlloc(int32_t DeviceId, int64_t Size, void *HstPtr);
int32_t dataDelete(int32_t DeviceId, void *TgtPtr);
int32_t dataSubmit(int32_t DeviceId, void *TgtPtr, void *HstPtr,
int64_t Size);
int32_t dataRetrieve(int32_t DeviceId, void *HstPtr, void *TgtPtr,
int64_t Size);
int32_t isDataExchangeable(int32_t SrcDevId, int32_t DstDevId);
int32_t dataExchange(int32_t SrcDevId, void *SrcPtr, int32_t DstDevId,
void *DstPtr, int64_t Size);
int32_t runTargetTeamRegion(int32_t DeviceId, void *TgtEntryPtr,
void **TgtArgs, ptrdiff_t *TgtOffsets,
int32_t ArgNum, int32_t TeamNum,
int32_t ThreadLimit, uint64_t LoopTripCount);
};
#endif

openmp/libomptarget/plugins/remote/src/Client.cpp

  • This file was deleted.
//===----------------- Client.cpp - Client Implementation -----------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// gRPC (Client) for the remote plugin.
//
//===----------------------------------------------------------------------===//
#include <cmath>
#include "Client.h"
#include "omptarget.h"
#include "openmp.pb.h"
using namespace std::chrono;
using grpc::ClientContext;
using grpc::ClientReader;
using grpc::ClientWriter;
using grpc::Status;
template <typename Fn1, typename Fn2, typename TReturn>
auto RemoteOffloadClient::remoteCall(Fn1 Preprocessor, Fn2 Postprocessor,
TReturn ErrorValue, bool CanTimeOut) {
ArenaAllocatorLock->lock();
if (Arena->SpaceAllocated() >= MaxSize)
Arena->Reset();
ArenaAllocatorLock->unlock();
ClientContext Context;
if (CanTimeOut) {
auto Deadline =
std::chrono::system_clock::now() + std::chrono::seconds(Timeout);
Context.set_deadline(Deadline);
}
Status RPCStatus;
auto Reply = Preprocessor(RPCStatus, Context);
if (!RPCStatus.ok()) {
CLIENT_DBG("%s", RPCStatus.error_message().c_str())
} else {
return Postprocessor(Reply);
}
CLIENT_DBG("Failed")
return ErrorValue;
}
int32_t RemoteOffloadClient::shutdown(void) {
ClientContext Context;
Null Request;
I32 Reply;
CLIENT_DBG("Shutting down server.")
auto Status = Stub->Shutdown(&Context, Request, &Reply);
if (Status.ok())
return Reply.number();
return 1;
}
int32_t RemoteOffloadClient::registerLib(__tgt_bin_desc *Desc) {
return remoteCall(
/* Preprocessor */
[&](auto &RPCStatus, auto &Context) {
auto *Request = protobuf::Arena::CreateMessage<TargetBinaryDescription>(
Arena.get());
auto *Reply = protobuf::Arena::CreateMessage<I32>(Arena.get());
loadTargetBinaryDescription(Desc, *Request);
Request->set_bin_ptr((uint64_t)Desc);
RPCStatus = Stub->RegisterLib(&Context, *Request, Reply);
return Reply;
},
/* Postprocessor */
[&](const auto &Reply) {
if (Reply->number() == 0) {
CLIENT_DBG("Registered library")
return 0;
}
return 1;
},
/* Error Value */ 1);
}
int32_t RemoteOffloadClient::unregisterLib(__tgt_bin_desc *Desc) {
return remoteCall(
/* Preprocessor */
[&](auto &RPCStatus, auto &Context) {
auto *Request = protobuf::Arena::CreateMessage<Pointer>(Arena.get());
auto *Reply = protobuf::Arena::CreateMessage<I32>(Arena.get());
Request->set_number((uint64_t)Desc);
RPCStatus = Stub->UnregisterLib(&Context, *Request, Reply);
return Reply;
},
/* Postprocessor */
[&](const auto &Reply) {
if (Reply->number() == 0) {
CLIENT_DBG("Unregistered library")
return 0;
}
CLIENT_DBG("Failed to unregister library")
return 1;
},
/* Error Value */ 1);
}
int32_t RemoteOffloadClient::isValidBinary(__tgt_device_image *Image) {
return remoteCall(
/* Preprocessor */
[&](auto &RPCStatus, auto &Context) {
auto *Request =
protobuf::Arena::CreateMessage<TargetDeviceImagePtr>(Arena.get());
auto *Reply = protobuf::Arena::CreateMessage<I32>(Arena.get());
Request->set_image_ptr((uint64_t)Image->ImageStart);
auto *EntryItr = Image->EntriesBegin;
while (EntryItr != Image->EntriesEnd)
Request->add_entry_ptrs((uint64_t)EntryItr++);
RPCStatus = Stub->IsValidBinary(&Context, *Request, Reply);
return Reply;
},
/* Postprocessor */
[&](const auto &Reply) {
if (Reply->number()) {
CLIENT_DBG("Validated binary")
} else {
CLIENT_DBG("Could not validate binary")
}
return Reply->number();
},
/* Error Value */ 0);
}
int32_t RemoteOffloadClient::getNumberOfDevices() {
return remoteCall(
/* Preprocessor */
[&](Status &RPCStatus, ClientContext &Context) {
auto *Request = protobuf::Arena::CreateMessage<Null>(Arena.get());
auto *Reply = protobuf::Arena::CreateMessage<I32>(Arena.get());
RPCStatus = Stub->GetNumberOfDevices(&Context, *Request, Reply);
return Reply;
},
/* Postprocessor */
[&](const auto &Reply) {
if (Reply->number()) {
CLIENT_DBG("Found %d devices", Reply->number())
} else {
CLIENT_DBG("Could not get the number of devices")
}
return Reply->number();
},
/*Error Value*/ -1);
}
int32_t RemoteOffloadClient::initDevice(int32_t DeviceId) {
return remoteCall(
/* Preprocessor */
[&](auto &RPCStatus, auto &Context) {
auto *Request = protobuf::Arena::CreateMessage<I32>(Arena.get());
auto *Reply = protobuf::Arena::CreateMessage<I32>(Arena.get());
Request->set_number(DeviceId);
RPCStatus = Stub->InitDevice(&Context, *Request, Reply);
return Reply;
},
/* Postprocessor */
[&](const auto &Reply) {
if (!Reply->number()) {
CLIENT_DBG("Initialized device %d", DeviceId)
} else {
CLIENT_DBG("Could not initialize device %d", DeviceId)
}
return Reply->number();
},
/* Error Value */ -1);
}
int32_t RemoteOffloadClient::initRequires(int64_t RequiresFlags) {
return remoteCall(
/* Preprocessor */
[&](auto &RPCStatus, auto &Context) {
auto *Request = protobuf::Arena::CreateMessage<I64>(Arena.get());
auto *Reply = protobuf::Arena::CreateMessage<I32>(Arena.get());
Request->set_number(RequiresFlags);
RPCStatus = Stub->InitRequires(&Context, *Request, Reply);
return Reply;
},
/* Postprocessor */
[&](const auto &Reply) {
if (Reply->number()) {
CLIENT_DBG("Initialized requires")
} else {
CLIENT_DBG("Could not initialize requires")
}
return Reply->number();
},
/* Error Value */ -1);
}
__tgt_target_table *RemoteOffloadClient::loadBinary(int32_t DeviceId,
__tgt_device_image *Image) {
return remoteCall(
/* Preprocessor */
[&](auto &RPCStatus, auto &Context) {
auto *ImageMessage =
protobuf::Arena::CreateMessage<Binary>(Arena.get());
auto *Reply = protobuf::Arena::CreateMessage<TargetTable>(Arena.get());
ImageMessage->set_image_ptr((uint64_t)Image->ImageStart);
ImageMessage->set_device_id(DeviceId);
RPCStatus = Stub->LoadBinary(&Context, *ImageMessage, Reply);
return Reply;
},
/* Postprocessor */
[&](auto &Reply) {
if (Reply->entries_size() == 0) {
CLIENT_DBG("Could not load image %p onto device %d", Image, DeviceId)
return (__tgt_target_table *)nullptr;
}
DevicesToTables[DeviceId] = std::make_unique<__tgt_target_table>();
unloadTargetTable(*Reply, DevicesToTables[DeviceId].get(),
RemoteEntries[DeviceId]);
CLIENT_DBG("Loaded Image %p to device %d with %d entries", Image,
DeviceId, Reply->entries_size())
return DevicesToTables[DeviceId].get();
},
/* Error Value */ (__tgt_target_table *)nullptr,
/* CanTimeOut */ false);
}
int32_t RemoteOffloadClient::isDataExchangeable(int32_t SrcDevId,
int32_t DstDevId) {
return remoteCall(
/* Preprocessor */
[&](auto &RPCStatus, auto &Context) {
auto *Request = protobuf::Arena::CreateMessage<DevicePair>(Arena.get());
auto *Reply = protobuf::Arena::CreateMessage<I32>(Arena.get());
Request->set_src_dev_id(SrcDevId);
Request->set_dst_dev_id(DstDevId);
RPCStatus = Stub->IsDataExchangeable(&Context, *Request, Reply);
return Reply;
},
/* Postprocessor */
[&](auto &Reply) {
if (Reply->number()) {
CLIENT_DBG("Data is exchangeable between %d, %d", SrcDevId, DstDevId)
} else {
CLIENT_DBG("Data is not exchangeable between %d, %d", SrcDevId,
DstDevId)
}
return Reply->number();
},
/* Error Value */ -1);
}
void *RemoteOffloadClient::dataAlloc(int32_t DeviceId, int64_t Size,
void *HstPtr) {
return remoteCall(
/* Preprocessor */
[&](auto &RPCStatus, auto &Context) {
auto *Reply = protobuf::Arena::CreateMessage<Pointer>(Arena.get());
auto *Request = protobuf::Arena::CreateMessage<AllocData>(Arena.get());
Request->set_device_id(DeviceId);
Request->set_size(Size);
Request->set_hst_ptr((uint64_t)HstPtr);
RPCStatus = Stub->DataAlloc(&Context, *Request, Reply);
return Reply;
},
/* Postprocessor */
[&](auto &Reply) {
if (Reply->number()) {
CLIENT_DBG("Allocated %ld bytes on device %d at %p", Size, DeviceId,
(void *)Reply->number())
} else {
CLIENT_DBG("Could not allocate %ld bytes on device %d at %p", Size,
DeviceId, (void *)Reply->number())
}
return (void *)Reply->number();
},
/* Error Value */ (void *)nullptr);
}
int32_t RemoteOffloadClient::dataSubmit(int32_t DeviceId, void *TgtPtr,
void *HstPtr, int64_t Size) {
return remoteCall(
/* Preprocessor */
[&](auto &RPCStatus, auto &Context) {
auto *Reply = protobuf::Arena::CreateMessage<I32>(Arena.get());
std::unique_ptr<ClientWriter<SubmitData>> Writer(
Stub->DataSubmit(&Context, Reply));
if (Size > BlockSize) {
int64_t Start = 0, End = BlockSize;
for (auto I = 0; I < ceil((float)Size / BlockSize); I++) {
auto *Request =
protobuf::Arena::CreateMessage<SubmitData>(Arena.get());
Request->set_device_id(DeviceId);
Request->set_data((char *)HstPtr + Start, End - Start);
Request->set_hst_ptr((uint64_t)HstPtr);
Request->set_tgt_ptr((uint64_t)TgtPtr);
Request->set_start(Start);
Request->set_size(Size);
if (!Writer->Write(*Request)) {
CLIENT_DBG("Broken stream when submitting data")
Reply->set_number(0);
return Reply;
}
Start += BlockSize;
End += BlockSize;
if (End >= Size)
End = Size;
}
} else {
auto *Request =
protobuf::Arena::CreateMessage<SubmitData>(Arena.get());
Request->set_device_id(DeviceId);
Request->set_data(HstPtr, Size);
Request->set_hst_ptr((uint64_t)HstPtr);
Request->set_tgt_ptr((uint64_t)TgtPtr);
Request->set_start(0);
Request->set_size(Size);
if (!Writer->Write(*Request)) {
CLIENT_DBG("Broken stream when submitting data")
Reply->set_number(0);
return Reply;
}
}
Writer->WritesDone();
RPCStatus = Writer->Finish();
return Reply;
},
/* Postprocessor */
[&](auto &Reply) {
if (!Reply->number()) {
CLIENT_DBG(" submitted %ld bytes on device %d at %p", Size, DeviceId,
TgtPtr)
} else {
CLIENT_DBG("Could not async submit %ld bytes on device %d at %p",
Size, DeviceId, TgtPtr)
}
return Reply->number();
},
/* Error Value */ -1,
/* CanTimeOut */ false);
}
int32_t RemoteOffloadClient::dataRetrieve(int32_t DeviceId, void *HstPtr,
void *TgtPtr, int64_t Size) {
return remoteCall(
/* Preprocessor */
[&](auto &RPCStatus, auto &Context) {
auto *Request =
protobuf::Arena::CreateMessage<RetrieveData>(Arena.get());
Request->set_device_id(DeviceId);
Request->set_size(Size);
Request->set_hst_ptr((int64_t)HstPtr);
Request->set_tgt_ptr((int64_t)TgtPtr);
auto *Reply = protobuf::Arena::CreateMessage<Data>(Arena.get());
std::unique_ptr<ClientReader<Data>> Reader(
Stub->DataRetrieve(&Context, *Request));
Reader->WaitForInitialMetadata();
while (Reader->Read(Reply)) {
if (Reply->ret()) {
CLIENT_DBG("Could not async retrieve %ld bytes on device %d at %p "
"for %p",
Size, DeviceId, TgtPtr, HstPtr)
return Reply;
}
if (Reply->start() == 0 && Reply->size() == Reply->data().size()) {
memcpy(HstPtr, Reply->data().data(), Reply->data().size());
return Reply;
}
memcpy((void *)((char *)HstPtr + Reply->start()),
Reply->data().data(), Reply->data().size());
}
RPCStatus = Reader->Finish();
return Reply;
},
/* Postprocessor */
[&](auto &Reply) {
if (!Reply->ret()) {
CLIENT_DBG("Retrieved %ld bytes on Device %d", Size, DeviceId)
} else {
CLIENT_DBG("Could not async retrieve %ld bytes on Device %d", Size,
DeviceId)
}
return Reply->ret();
},
/* Error Value */ -1,
/* CanTimeOut */ false);
}
int32_t RemoteOffloadClient::dataExchange(int32_t SrcDevId, void *SrcPtr,
int32_t DstDevId, void *DstPtr,
int64_t Size) {
return remoteCall(
/* Preprocessor */
[&](auto &RPCStatus, auto &Context) {
auto *Reply = protobuf::Arena::CreateMessage<I32>(Arena.get());
auto *Request =
protobuf::Arena::CreateMessage<ExchangeData>(Arena.get());
Request->set_src_dev_id(SrcDevId);
Request->set_src_ptr((uint64_t)SrcPtr);
Request->set_dst_dev_id(DstDevId);
Request->set_dst_ptr((uint64_t)DstPtr);
Request->set_size(Size);
RPCStatus = Stub->DataExchange(&Context, *Request, Reply);
return Reply;
},
/* Postprocessor */
[&](auto &Reply) {
if (Reply->number()) {
CLIENT_DBG(
"Exchanged %ld bytes on device %d at %p for %p on device %d",
Size, SrcDevId, SrcPtr, DstPtr, DstDevId)
} else {
CLIENT_DBG("Could not exchange %ld bytes on device %d at %p for %p "
"on device %d",
Size, SrcDevId, SrcPtr, DstPtr, DstDevId)
}
return Reply->number();
},
/* Error Value */ -1);
}
int32_t RemoteOffloadClient::dataDelete(int32_t DeviceId, void *TgtPtr) {
return remoteCall(
/* Preprocessor */
[&](auto &RPCStatus, auto &Context) {
auto *Reply = protobuf::Arena::CreateMessage<I32>(Arena.get());
auto *Request = protobuf::Arena::CreateMessage<DeleteData>(Arena.get());
Request->set_device_id(DeviceId);
Request->set_tgt_ptr((uint64_t)TgtPtr);
RPCStatus = Stub->DataDelete(&Context, *Request, Reply);
return Reply;
},
/* Postprocessor */
[&](auto &Reply) {
if (!Reply->number()) {
CLIENT_DBG("Deleted data at %p on device %d", TgtPtr, DeviceId)
} else {
CLIENT_DBG("Could not delete data at %p on device %d", TgtPtr,
DeviceId)
}
return Reply->number();
},
/* Error Value */ -1);
}
int32_t RemoteOffloadClient::runTargetRegion(int32_t DeviceId,
void *TgtEntryPtr, void **TgtArgs,
ptrdiff_t *TgtOffsets,
int32_t ArgNum) {
return remoteCall(
/* Preprocessor */
[&](auto &RPCStatus, auto &Context) {
auto *Reply = protobuf::Arena::CreateMessage<I32>(Arena.get());
auto *Request =
protobuf::Arena::CreateMessage<TargetRegion>(Arena.get());
Request->set_device_id(DeviceId);
Request->set_tgt_entry_ptr(
(uint64_t)RemoteEntries[DeviceId][TgtEntryPtr]);
char **ArgPtr = (char **)TgtArgs;
for (auto I = 0; I < ArgNum; I++, ArgPtr++)
Request->add_tgt_args((uint64_t)*ArgPtr);
char *OffsetPtr = (char *)TgtOffsets;
for (auto I = 0; I < ArgNum; I++, OffsetPtr++)
Request->add_tgt_offsets((uint64_t)*OffsetPtr);
Request->set_arg_num(ArgNum);
RPCStatus = Stub->RunTargetRegion(&Context, *Request, Reply);
return Reply;
},
/* Postprocessor */
[&](auto &Reply) {
if (!Reply->number()) {
CLIENT_DBG("Ran target region async on device %d", DeviceId)
} else {
CLIENT_DBG("Could not run target region async on device %d", DeviceId)
}
return Reply->number();
},
/* Error Value */ -1,
/* CanTimeOut */ false);
}
int32_t RemoteOffloadClient::runTargetTeamRegion(
int32_t DeviceId, void *TgtEntryPtr, void **TgtArgs, ptrdiff_t *TgtOffsets,
int32_t ArgNum, int32_t TeamNum, int32_t ThreadLimit,
uint64_t LoopTripcount) {
return remoteCall(
/* Preprocessor */
[&](auto &RPCStatus, auto &Context) {
auto *Reply = protobuf::Arena::CreateMessage<I32>(Arena.get());
auto *Request =
protobuf::Arena::CreateMessage<TargetTeamRegion>(Arena.get());
Request->set_device_id(DeviceId);
Request->set_tgt_entry_ptr(
(uint64_t)RemoteEntries[DeviceId][TgtEntryPtr]);
char **ArgPtr = (char **)TgtArgs;
for (auto I = 0; I < ArgNum; I++, ArgPtr++) {
Request->add_tgt_args((uint64_t)*ArgPtr);
}
char *OffsetPtr = (char *)TgtOffsets;
for (auto I = 0; I < ArgNum; I++, OffsetPtr++)
Request->add_tgt_offsets((uint64_t)*OffsetPtr);
Request->set_arg_num(ArgNum);
Request->set_team_num(TeamNum);
Request->set_thread_limit(ThreadLimit);
Request->set_loop_tripcount(LoopTripcount);
RPCStatus = Stub->RunTargetTeamRegion(&Context, *Request, Reply);
return Reply;
},
/* Postprocessor */
[&](auto &Reply) {
if (!Reply->number()) {
CLIENT_DBG("Ran target team region async on device %d", DeviceId)
} else {
CLIENT_DBG("Could not run target team region async on device %d",
DeviceId)
}
return Reply->number();
},
/* Error Value */ -1,
/* CanTimeOut */ false);
}
int32_t RemoteClientManager::shutdown(void) {
int32_t Ret = 0;
for (auto &Client : Clients)
Ret &= Client.shutdown();
return Ret;
}
int32_t RemoteClientManager::registerLib(__tgt_bin_desc *Desc) {
int32_t Ret = 0;
for (auto &Client : Clients)
Ret &= Client.registerLib(Desc);
return Ret;
}
int32_t RemoteClientManager::unregisterLib(__tgt_bin_desc *Desc) {
int32_t Ret = 0;
for (auto &Client : Clients)
Ret &= Client.unregisterLib(Desc);
return Ret;
}
int32_t RemoteClientManager::isValidBinary(__tgt_device_image *Image) {
int32_t ClientIdx = 0;
for (auto &Client : Clients) {
if (auto Ret = Client.isValidBinary(Image))
return Ret;
ClientIdx++;
}
return 0;
}
int32_t RemoteClientManager::getNumberOfDevices() {
auto ClientIdx = 0;
for (auto &Client : Clients) {
if (auto NumDevices = Client.getNumberOfDevices()) {
Devices.push_back(NumDevices);
}
ClientIdx++;
}
return std::accumulate(Devices.begin(), Devices.end(), 0);
}
std::pair<int32_t, int32_t> RemoteClientManager::mapDeviceId(int32_t DeviceId) {
for (size_t ClientIdx = 0; ClientIdx < Devices.size(); ClientIdx++) {
if (DeviceId < Devices[ClientIdx])
return {ClientIdx, DeviceId};
DeviceId -= Devices[ClientIdx];
}
return {-1, -1};
}
int32_t RemoteClientManager::initDevice(int32_t DeviceId) {
int32_t ClientIdx, DeviceIdx;
std::tie(ClientIdx, DeviceIdx) = mapDeviceId(DeviceId);
return Clients[ClientIdx].initDevice(DeviceIdx);
}
int32_t RemoteClientManager::initRequires(int64_t RequiresFlags) {
for (auto &Client : Clients)
Client.initRequires(RequiresFlags);
return RequiresFlags;
}
__tgt_target_table *RemoteClientManager::loadBinary(int32_t DeviceId,
__tgt_device_image *Image) {
int32_t ClientIdx, DeviceIdx;
std::tie(ClientIdx, DeviceIdx) = mapDeviceId(DeviceId);
return Clients[ClientIdx].loadBinary(DeviceIdx, Image);
}
int32_t RemoteClientManager::isDataExchangeable(int32_t SrcDevId,
int32_t DstDevId) {
int32_t SrcClientIdx, SrcDeviceIdx, DstClientIdx, DstDeviceIdx;
std::tie(SrcClientIdx, SrcDeviceIdx) = mapDeviceId(SrcDevId);
std::tie(DstClientIdx, DstDeviceIdx) = mapDeviceId(DstDevId);
return Clients[SrcClientIdx].isDataExchangeable(SrcDeviceIdx, DstDeviceIdx);
}
void *RemoteClientManager::dataAlloc(int32_t DeviceId, int64_t Size,
void *HstPtr) {
int32_t ClientIdx, DeviceIdx;
std::tie(ClientIdx, DeviceIdx) = mapDeviceId(DeviceId);
return Clients[ClientIdx].dataAlloc(DeviceIdx, Size, HstPtr);
}
int32_t RemoteClientManager::dataDelete(int32_t DeviceId, void *TgtPtr) {
int32_t ClientIdx, DeviceIdx;
std::tie(ClientIdx, DeviceIdx) = mapDeviceId(DeviceId);
return Clients[ClientIdx].dataDelete(DeviceIdx, TgtPtr);
}
int32_t RemoteClientManager::dataSubmit(int32_t DeviceId, void *TgtPtr,
void *HstPtr, int64_t Size) {
int32_t ClientIdx, DeviceIdx;
std::tie(ClientIdx, DeviceIdx) = mapDeviceId(DeviceId);
return Clients[ClientIdx].dataSubmit(DeviceIdx, TgtPtr, HstPtr, Size);
}
int32_t RemoteClientManager::dataRetrieve(int32_t DeviceId, void *HstPtr,
void *TgtPtr, int64_t Size) {
int32_t ClientIdx, DeviceIdx;
std::tie(ClientIdx, DeviceIdx) = mapDeviceId(DeviceId);
return Clients[ClientIdx].dataRetrieve(DeviceIdx, HstPtr, TgtPtr, Size);
}
int32_t RemoteClientManager::dataExchange(int32_t SrcDevId, void *SrcPtr,
int32_t DstDevId, void *DstPtr,
int64_t Size) {
int32_t SrcClientIdx, SrcDeviceIdx, DstClientIdx, DstDeviceIdx;
std::tie(SrcClientIdx, SrcDeviceIdx) = mapDeviceId(SrcDevId);
std::tie(DstClientIdx, DstDeviceIdx) = mapDeviceId(DstDevId);
return Clients[SrcClientIdx].dataExchange(SrcDeviceIdx, SrcPtr, DstDeviceIdx,
DstPtr, Size);
}
int32_t RemoteClientManager::runTargetRegion(int32_t DeviceId,
void *TgtEntryPtr, void **TgtArgs,
ptrdiff_t *TgtOffsets,
int32_t ArgNum) {
int32_t ClientIdx, DeviceIdx;
std::tie(ClientIdx, DeviceIdx) = mapDeviceId(DeviceId);
return Clients[ClientIdx].runTargetRegion(DeviceIdx, TgtEntryPtr, TgtArgs,
TgtOffsets, ArgNum);
}
int32_t RemoteClientManager::runTargetTeamRegion(
int32_t DeviceId, void *TgtEntryPtr, void **TgtArgs, ptrdiff_t *TgtOffsets,
int32_t ArgNum, int32_t TeamNum, int32_t ThreadLimit,
uint64_t LoopTripCount) {
int32_t ClientIdx, DeviceIdx;
std::tie(ClientIdx, DeviceIdx) = mapDeviceId(DeviceId);
return Clients[ClientIdx].runTargetTeamRegion(DeviceIdx, TgtEntryPtr, TgtArgs,
TgtOffsets, ArgNum, TeamNum,
ThreadLimit, LoopTripCount);
}

openmp/libomptarget/plugins/remote/src/rtl.cpp

  • This file was deleted.
//===--------------------- rtl.cpp - Remote RTL Plugin --------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// RTL for Host.
//
//===----------------------------------------------------------------------===//
#include <cstddef>
#include <memory>
#include <string>
#include "Client.h"
#include "Utils.h"
#include "omptarget.h"
#include "omptargetplugin.h"
#define TARGET_NAME RPC
#define DEBUG_PREFIX "Target " GETNAME(TARGET_NAME) " RTL"
RemoteClientManager *Manager;
__attribute__((constructor(101))) void initRPC() {
DP("Init RPC library!\n");
Manager = new RemoteClientManager();
}
__attribute__((destructor(101))) void deinitRPC() {
Manager->shutdown(); // TODO: Error handle shutting down
DP("Deinit RPC library!\n");
delete Manager;
}
// Exposed library API function
#ifdef __cplusplus
extern "C" {
#endif
int32_t __tgt_rtl_register_lib(__tgt_bin_desc *Desc) {
return Manager->registerLib(Desc);
}
int32_t __tgt_rtl_unregister_lib(__tgt_bin_desc *Desc) {
return Manager->unregisterLib(Desc);
}
int32_t __tgt_rtl_is_valid_binary(__tgt_device_image *Image) {
return Manager->isValidBinary(Image);
}
int32_t __tgt_rtl_number_of_devices() { return Manager->getNumberOfDevices(); }
int32_t __tgt_rtl_init_device(int32_t DeviceId) {
return Manager->initDevice(DeviceId);
}
int64_t __tgt_rtl_init_requires(int64_t RequiresFlags) {
return Manager->initRequires(RequiresFlags);
}
__tgt_target_table *__tgt_rtl_load_binary(int32_t DeviceId,
__tgt_device_image *Image) {
return Manager->loadBinary(DeviceId, (__tgt_device_image *)Image);
}
int32_t __tgt_rtl_is_data_exchangable(int32_t SrcDevId, int32_t DstDevId) {
return Manager->isDataExchangeable(SrcDevId, DstDevId);
}
void *__tgt_rtl_data_alloc(int32_t DeviceId, int64_t Size, void *HstPtr,
int32_t Kind) {
if (Kind != TARGET_ALLOC_DEFAULT) {
REPORT("Invalid target data allocation kind or requested allocator not "
"implemented yet\n");
return NULL;
}
return Manager->dataAlloc(DeviceId, Size, HstPtr);
}
int32_t __tgt_rtl_data_submit(int32_t DeviceId, void *TgtPtr, void *HstPtr,
int64_t Size) {
return Manager->dataSubmit(DeviceId, TgtPtr, HstPtr, Size);
}
int32_t __tgt_rtl_data_retrieve(int32_t DeviceId, void *HstPtr, void *TgtPtr,
int64_t Size) {
return Manager->dataRetrieve(DeviceId, HstPtr, TgtPtr, Size);
}
int32_t __tgt_rtl_data_delete(int32_t DeviceId, void *TgtPtr, int32_t) {
return Manager->dataDelete(DeviceId, TgtPtr);
}
int32_t __tgt_rtl_data_exchange(int32_t SrcDevId, void *SrcPtr,
int32_t DstDevId, void *DstPtr, int64_t Size) {
return Manager->dataExchange(SrcDevId, SrcPtr, DstDevId, DstPtr, Size);
}
int32_t __tgt_rtl_launch_kernel(int32_t DeviceId, void *TgtEntryPtr,
void **TgtArgs, ptrdiff_t *TgtOffsets,
KernelArgsTy *KernelArgs,
__tgt_async_info *AsyncInfoPtr) {
assert(!KernelArgs->NumTeams[1] && !KernelArgs->NumTeams[2] &&
!KernelArgs->ThreadLimit[1] && !KernelArgs->ThreadLimit[2] &&
"Only one dimensional kernels supported.");
return Manager->runTargetTeamRegion(
DeviceId, TgtEntryPtr, TgtArgs, TgtOffsets, KernelArgs->NumArgs,
KernelArgs->NumTeams[0], KernelArgs->ThreadLimit[0],
KernelArgs->Tripcount);
}
// Exposed library API function
#ifdef __cplusplus
}
#endif

openmp/libomptarget/plugins/ve/CMakeLists.txt

  • This file was deleted.
##===----------------------------------------------------------------------===##
#
# Build a plugin for a NEC Aurora machine if available. (Can also run on host)
#
##===----------------------------------------------------------------------===##
if(${LIBOMPTARGET_DEP_VEO_FOUND})
libomptarget_say("Building SX-Aurora VE offloading plugin.")
set(additional_libs "")
set(additional_libs ${LIBOMPTARGET_DEP_VEO_LIBRARIES}
${LIBOMPTARGET_DEP_VEOSINFO_LIBRARIES}
${additional_libs})
set(tmachine_name "ve")
set(tmachine_libname "ve")
set(tmachine_triple "ve-unknown-linux-unknown")
set(elf_machine_id 251)
# Define macro to be used as prefix of the runtime messages for this target.
add_definitions("-DTARGET_NAME=${tmachine_name}")
# Define macro with the ELF ID for this target.
add_definitions("-DTARGET_ELF_ID=${elf_machine_id}")
add_llvm_library("omptarget.rtl.${tmachine_libname}"
SHARED
${CMAKE_CURRENT_SOURCE_DIR}/src/rtl.cpp
ADDITIONAL_HEADER_DIRS
${LIBOMPTARGET_INCLUDE_DIR}
${LIBOMPTARGET_DEP_VEO_INCLUDE_DIR}
LINK_LIBS
PRIVATE
elf_common
${LIBOMPTARGET_DEP_LIBFFI_LIBRARIES}
${additional_libs}
"-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports -Wl,-z,defs"
NO_INSTALL_RPATH
)
# Install plugin under the lib destination folder.
install(TARGETS "omptarget.rtl.${tmachine_libname}" LIBRARY DESTINATION "${OPENMP_INSTALL_LIBDIR}")
set_target_properties("omptarget.rtl.${tmachine_libname}" PROPERTIES
INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/.."
CXX_VISIBILITY_PRESET protected)
target_include_directories("omptarget.rtl.${tmachine_libname}" PRIVATE
${LIBOMPTARGET_INCLUDE_DIR}
${LIBOMPTARGET_DEP_VEO_INCLUDE_DIR})
target_link_libraries(
"omptarget.rtl.${tmachine_libname}"
elf_common
${LIBOMPTARGET_DEP_LIBFFI_LIBRARIES}
${additional_libs}
"-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports -Wl,-z,defs")
# Report to the parent scope that we are building a plugin.
set(LIBOMPTARGET_SYSTEM_TARGETS
"${LIBOMPTARGET_SYSTEM_TARGETS} ${tmachine_triple}" PARENT_SCOPE)
else()
libomptarget_say("Not building nec-aurora plugin: libveo or libveosinfo not found.")
endif()

openmp/libomptarget/plugins/ve/src/rtl.cpp

  • This file was deleted.
//===-RTLs/nec-aurora/src/rtl.cpp - Target RTLs Implementation - C++ -*-======//
//
// The LLVM Compiler Infrastructure
//
// This file is dual licensed under the MIT and the University of Illinois Open
// Source Licenses. See LICENSE.txt for details.
//
//===----------------------------------------------------------------------===//
//
// RTL for NEC Aurora TSUBASA machines
//
//===----------------------------------------------------------------------===//
#include <algorithm>
#include <cassert>
#include <cerrno>
#include <cstring>
#include <list>
#include <stdlib.h>
#include <string>
#include <sys/stat.h>
#include <ve_offload.h>
#include <vector>
#include <veosinfo/veosinfo.h>
#include "Debug.h"
#include "omptargetplugin.h"
#ifndef TARGET_NAME
#define TARGET_NAME VE
#endif
#define DEBUG_PREFIX "Target " GETNAME(TARGET_NAME) " RTL"
#ifndef TARGET_ELF_ID
#define TARGET_ELF_ID 0
#endif
#include "elf_common.h"
struct DynLibTy {
char *FileName;
uint64_t VeoLibHandle;
};
/// Keep entries table per device.
struct FuncOrGblEntryTy {
__tgt_target_table Table;
std::vector<__tgt_offload_entry> Entries;
};
class RTLDeviceInfoTy {
std::vector<std::list<FuncOrGblEntryTy>> FuncOrGblEntry;
public:
std::vector<struct veo_proc_handle *> ProcHandles;
std::vector<struct veo_thr_ctxt *> Contexts;
std::vector<uint64_t> LibraryHandles;
std::list<DynLibTy> DynLibs;
// Maps OpenMP device Ids to Ve nodeids
std::vector<int> NodeIds;
void buildOffloadTableFromHost(int32_t device_id, uint64_t VeoLibHandle,
__tgt_offload_entry *HostBegin,
__tgt_offload_entry *HostEnd) {
FuncOrGblEntry[device_id].emplace_back();
std::vector<__tgt_offload_entry> &T =
FuncOrGblEntry[device_id].back().Entries;
T.clear();
for (__tgt_offload_entry *i = HostBegin; i != HostEnd; ++i) {
char *SymbolName = i->name;
// we have not enough access to the target memory to conveniently parse
// the offload table there so we need to lookup every symbol with the host
// table
DP("Looking up symbol: %s\n", SymbolName);
uint64_t SymbolTargetAddr =
veo_get_sym(ProcHandles[device_id], VeoLibHandle, SymbolName);
__tgt_offload_entry Entry;
if (!SymbolTargetAddr) {
DP("Symbol %s not found in target image\n", SymbolName);
Entry = {NULL, NULL, 0, 0, 0};
} else {
DP("Found symbol %s successfully in target image (addr: %p)\n",
SymbolName, reinterpret_cast<void *>(SymbolTargetAddr));
Entry = {reinterpret_cast<void *>(SymbolTargetAddr), i->name, i->size,
i->flags, 0};
}
T.push_back(Entry);
}
FuncOrGblEntry[device_id].back().Table.EntriesBegin = &T.front();
FuncOrGblEntry[device_id].back().Table.EntriesEnd = &T.back() + 1;
}
__tgt_target_table *getOffloadTable(int32_t device_id) {
return &FuncOrGblEntry[device_id].back().Table;
}
RTLDeviceInfoTy() {
struct ve_nodeinfo node_info;
ve_node_info(&node_info);
// Build a predictable mapping between VE node ids and OpenMP device ids.
// This is necessary, because nodes can be missing or offline and (active)
// node ids are thus not consecutive. The entries in ve_nodeinfo may also
// not be in the order of their node ids.
for (int i = 0; i < node_info.total_node_count; ++i) {
if (node_info.status[i] == 0) {
NodeIds.push_back(node_info.nodeid[i]);
}
}
// Because the entries in ve_nodeinfo may not be in the order of their node
// ids, we sort NodeIds to get a predictable mapping.
std::sort(NodeIds.begin(), NodeIds.end());
int NumDevices = NodeIds.size();
DP("Found %i VE devices\n", NumDevices);
ProcHandles.resize(NumDevices, NULL);
Contexts.resize(NumDevices, NULL);
FuncOrGblEntry.resize(NumDevices);
LibraryHandles.resize(NumDevices);
}
~RTLDeviceInfoTy() {
for (auto &ctx : Contexts) {
if (ctx != NULL) {
if (veo_context_close(ctx) != 0) {
DP("Failed to close VEO context.\n");
}
}
}
for (auto &hdl : ProcHandles) {
if (hdl != NULL) {
veo_proc_destroy(hdl);
}
}
for (auto &lib : DynLibs) {
if (lib.FileName) {
remove(lib.FileName);
}
}
}
};
static RTLDeviceInfoTy DeviceInfo;
static int target_run_function_wait(uint32_t DeviceID, uint64_t FuncAddr,
struct veo_args *args, uint64_t *RetVal) {
DP("Running function with entry point %p\n",
reinterpret_cast<void *>(FuncAddr));
uint64_t RequestHandle =
veo_call_async(DeviceInfo.Contexts[DeviceID], FuncAddr, args);
if (RequestHandle == VEO_REQUEST_ID_INVALID) {
DP("Execution of entry point %p failed\n",
reinterpret_cast<void *>(FuncAddr));
return OFFLOAD_FAIL;
}
DP("Function at address %p called (VEO request ID: %" PRIu64 ")\n",
reinterpret_cast<void *>(FuncAddr), RequestHandle);
int ret = veo_call_wait_result(DeviceInfo.Contexts[DeviceID], RequestHandle,
RetVal);
if (ret != 0) {
DP("Waiting for entry point %p failed (Error code %d)\n",
reinterpret_cast<void *>(FuncAddr), ret);
return OFFLOAD_FAIL;
}
return OFFLOAD_SUCCESS;
}
// Return the number of available devices of the type supported by the
// target RTL.
int32_t __tgt_rtl_number_of_devices(void) { return DeviceInfo.NodeIds.size(); }
// Return an integer different from zero if the provided device image can be
// supported by the runtime. The functionality is similar to comparing the
// result of __tgt__rtl__load__binary to NULL. However, this is meant to be a
// lightweight query to determine if the RTL is suitable for an image without
// having to load the library, which can be expensive.
int32_t __tgt_rtl_is_valid_binary(__tgt_device_image *Image) {
#if TARGET_ELF_ID < 1
return 0;
#else
return elf_check_machine(Image, TARGET_ELF_ID);
#endif
}
// Initialize the specified device. In case of success return 0; otherwise
// return an error code.
int32_t __tgt_rtl_init_device(int32_t ID) {
DP("Available VEO version: %i\n", veo_api_version());
// At the moment we do not really initialize (i.e. create a process or
// context on) the device here, but in "__tgt_rtl_load_binary".
// The reason for this is, that, when we create a process for a statically
// linked binary, the VEO api needs us to already supply the binary (but we
// can load a dynamically linked binary later, after we create the process).
// At this stage, we cannot check if we have a dynamically or statically
// linked binary so we defer process creation until we know.
return OFFLOAD_SUCCESS;
}
// Pass an executable image section described by image to the specified
// device and prepare an address table of target entities. In case of error,
// return NULL. Otherwise, return a pointer to the built address table.
// Individual entries in the table may also be NULL, when the corresponding
// offload region is not supported on the target device.
__tgt_target_table *__tgt_rtl_load_binary(int32_t ID,
__tgt_device_image *Image) {
DP("Dev %d: load binary from " DPxMOD " image\n", ID,
DPxPTR(Image->ImageStart));
assert(ID >= 0 && "bad dev id");
size_t ImageSize = (size_t)Image->ImageEnd - (size_t)Image->ImageStart;
size_t NumEntries = (size_t)(Image->EntriesEnd - Image->EntriesBegin);
DP("Expecting to have %zd entries defined.\n", NumEntries);
// load dynamic library and get the entry points. We use the dl library
// to do the loading of the library, but we could do it directly to avoid the
// dump to the temporary file.
//
// 1) Create tmp file with the library contents.
// 2) Use dlopen to load the file and dlsym to retrieve the symbols.
char tmp_name[] = "/tmp/tmpfile_XXXXXX";
int tmp_fd = mkstemp(tmp_name);
if (tmp_fd == -1) {
return NULL;
}
FILE *ftmp = fdopen(tmp_fd, "wb");
if (!ftmp) {
DP("fdopen() for %s failed. Could not write target image\n", tmp_name);
return NULL;
}
fwrite(Image->ImageStart, ImageSize, 1, ftmp);
// at least for the static case we need to change the permissions
chmod(tmp_name, 0700);
DP("Wrote target image to %s. ImageSize=%zu\n", tmp_name, ImageSize);
fclose(ftmp);
// See comment in "__tgt_rtl_init_device"
bool is_dyn = true;
if (DeviceInfo.ProcHandles[ID] == NULL) {
struct veo_proc_handle *proc_handle;
is_dyn = elf_is_dynamic(Image);
// If we have a dynamically linked image, we create the process handle, then
// the thread, and then load the image.
// If we have a statically linked image, we need to create the process
// handle and load the image at the same time with veo_proc_create_static().
if (is_dyn) {
proc_handle = veo_proc_create(DeviceInfo.NodeIds[ID]);
if (!proc_handle) {
DP("veo_proc_create() failed for device %d\n", ID);
return NULL;
}
} else {
proc_handle = veo_proc_create_static(DeviceInfo.NodeIds[ID], tmp_name);
if (!proc_handle) {
DP("veo_proc_create_static() failed for device %d, image=%s\n", ID,
tmp_name);
return NULL;
}
}
DeviceInfo.ProcHandles[ID] = proc_handle;
}
if (DeviceInfo.Contexts[ID] == NULL) {
struct veo_thr_ctxt *ctx = veo_context_open(DeviceInfo.ProcHandles[ID]);
if (!ctx) {
DP("veo_context_open() failed: %s\n", std::strerror(errno));
return NULL;
}
DeviceInfo.Contexts[ID] = ctx;
}
DP("Aurora device successfully initialized with loaded binary: "
"proc_handle=%p, ctx=%p\n",
DeviceInfo.ProcHandles[ID], DeviceInfo.Contexts[ID]);
uint64_t LibHandle = 0UL;
if (is_dyn) {
LibHandle = veo_load_library(DeviceInfo.ProcHandles[ID], tmp_name);
if (!LibHandle) {
DP("veo_load_library() failed: LibHandle=%" PRIu64
" Name=%s. Set env VEORUN_BIN for static linked target code.\n",
LibHandle, tmp_name);
return NULL;
}
DP("Successfully loaded library dynamically\n");
} else {
DP("Symbol table is expected to have been created by "
"veo_create_proc_static()\n");
}
DynLibTy Lib = {tmp_name, LibHandle};
DeviceInfo.DynLibs.push_back(Lib);
DeviceInfo.LibraryHandles[ID] = LibHandle;
DeviceInfo.buildOffloadTableFromHost(ID, LibHandle, Image->EntriesBegin,
Image->EntriesEnd);
return DeviceInfo.getOffloadTable(ID);
}
// Allocate data on the particular target device, of the specified size.
// HostPtr is a address of the host data the allocated target data
// will be associated with (HostPtr may be NULL if it is not known at
// allocation time, like for example it would be for target data that
// is allocated by omp_target_alloc() API). Return address of the
// allocated data on the target that will be used by libomptarget.so to
// initialize the target data mapping structures. These addresses are
// used to generate a table of target variables to pass to
// __tgt_rtl_run_region(). The __tgt_rtl_data_alloc() returns NULL in
// case an error occurred on the target device.
void *__tgt_rtl_data_alloc(int32_t ID, int64_t Size, void *HostPtr,
int32_t kind) {
int ret;
uint64_t addr;
if (kind != TARGET_ALLOC_DEFAULT) {
REPORT("Invalid target data allocation kind or requested allocator not "
"implemented yet\n");
return NULL;
}
if (DeviceInfo.ProcHandles[ID] == NULL) {
struct veo_proc_handle *proc_handle;
proc_handle = veo_proc_create(DeviceInfo.NodeIds[ID]);
if (!proc_handle) {
DP("veo_proc_create() failed for device %d\n", ID);
return NULL;
}
DeviceInfo.ProcHandles[ID] = proc_handle;
DP("Aurora device successfully initialized: proc_handle=%p", proc_handle);
}
ret = veo_alloc_mem(DeviceInfo.ProcHandles[ID], &addr, Size);
DP("Allocate target memory: device=%d, target addr=%p, size=%" PRIu64 "\n",
ID, reinterpret_cast<void *>(addr), Size);
if (ret != 0) {
DP("veo_alloc_mem(%d, %p, %" PRIu64 ") failed with error code %d\n", ID,
reinterpret_cast<void *>(addr), Size, ret);
return NULL;
}
return reinterpret_cast<void *>(addr);
}
// Pass the data content to the target device using the target address.
// In case of success, return zero. Otherwise, return an error code.
int32_t __tgt_rtl_data_submit(int32_t ID, void *TargetPtr, void *HostPtr,
int64_t Size) {
int ret = veo_write_mem(DeviceInfo.ProcHandles[ID], (uint64_t)TargetPtr,
HostPtr, (size_t)Size);
if (ret != 0) {
DP("veo_write_mem() failed with error code %d\n", ret);
return OFFLOAD_FAIL;
}
return OFFLOAD_SUCCESS;
}
// Retrieve the data content from the target device using its address.
// In case of success, return zero. Otherwise, return an error code.
int32_t __tgt_rtl_data_retrieve(int32_t ID, void *HostPtr, void *TargetPtr,
int64_t Size) {
int ret = veo_read_mem(DeviceInfo.ProcHandles[ID], HostPtr,
(uint64_t)TargetPtr, Size);
if (ret != 0) {
DP("veo_read_mem() failed with error code %d\n", ret);
return OFFLOAD_FAIL;
}
return OFFLOAD_SUCCESS;
}
// De-allocate the data referenced by target ptr on the device. In case of
// success, return zero. Otherwise, return an error code.
int32_t __tgt_rtl_data_delete(int32_t ID, void *TargetPtr, int32_t) {
int ret = veo_free_mem(DeviceInfo.ProcHandles[ID], (uint64_t)TargetPtr);
if (ret != 0) {
DP("veo_free_mem() failed with error code %d\n", ret);
return OFFLOAD_FAIL;
}
return OFFLOAD_SUCCESS;
}
// Transfer control to the offloaded entry Entry on the target device.
// Args and Offsets are arrays of NumArgs size of target addresses and
// offsets. An offset should be added to the target address before passing it
// to the outlined function on device side. In case of success, return zero.
// Otherwise, return an error code.
int32_t __tgt_rtl_launch_kernel(int32_t DeviceId, void *TgtEntryPtr,
void **TgtArgs, ptrdiff_t *TgtOffsets,
KernelArgsTy *KernelArgs,
__tgt_async_info *AsyncInfoPtr) {
assert(!KernelArgs->NumTeams[1] && !KernelArgs->NumTeams[2] &&
!KernelArgs->ThreadLimit[1] && !KernelArgs->ThreadLimit[2] &&
"Only one dimensional kernels supported.");
int ret;
// ignore team num and thread limit.
std::vector<void *> ptrs(KernelArgs->NumArgs);
struct veo_args *TargetArgs;
TargetArgs = veo_args_alloc();
if (TargetArgs == NULL) {
DP("Could not allocate VEO args\n");
return OFFLOAD_FAIL;
}
for (int i = 0; i < KernelArgs->NumArgs; ++i) {
ret = veo_args_set_u64(TargetArgs, i, (intptr_t)Args[i]);
if (ret != 0) {
DP("veo_args_set_u64() has returned %d for argnum=%d and value %p\n", ret,
i, Args[i]);
return OFFLOAD_FAIL;
}
}
uint64_t RetVal;
if (target_run_function_wait(ID, reinterpret_cast<uint64_t>(Entry),
TargetArgs, &RetVal) != OFFLOAD_SUCCESS) {
veo_args_free(TargetArgs);
return OFFLOAD_FAIL;
}
veo_args_free(TargetArgs);
return OFFLOAD_SUCCESS;
}
int32_t __tgt_rtl_supports_empty_images() { return 1; }
// VEC plugin's internal InfoLevel.
std::atomic<uint32_t> InfoLevel;

openmp/libomptarget/src/rtl.cpp

Show All 28 Lines
using namespace llvm::omp::target; using namespace llvm::omp::target;
// List of all plugins that can support offloading. // List of all plugins that can support offloading.
static const char *RTLNames[] = { static const char *RTLNames[] = {
/* PowerPC target */ "libomptarget.rtl.ppc64", /* PowerPC target */ "libomptarget.rtl.ppc64",
/* x86_64 target */ "libomptarget.rtl.x86_64", /* x86_64 target */ "libomptarget.rtl.x86_64",
/* CUDA target */ "libomptarget.rtl.cuda", /* CUDA target */ "libomptarget.rtl.cuda",
/* AArch64 target */ "libomptarget.rtl.aarch64", /* AArch64 target */ "libomptarget.rtl.aarch64",
/* SX-Aurora VE target */ "libomptarget.rtl.ve",
/* AMDGPU target */ "libomptarget.rtl.amdgpu", /* AMDGPU target */ "libomptarget.rtl.amdgpu",
/* Remote target */ "libomptarget.rtl.rpc",
}; };
PluginManager *PM; PluginManager *PM;
static char *ProfileTraceFile = nullptr; static char *ProfileTraceFile = nullptr;
#ifdef OMPT_SUPPORT #ifdef OMPT_SUPPORT
extern void InitOmptLibomp(); extern void InitOmptLibomp();
▲ Show 20 Lines • Show All 557 Lines • Show Last 20 Lines
openmp/libomptarget/plugins/remote/src/Client.h