[Libomptarget] Experimental Remote Plugin Fixes

[atmnpatel]

Summary

D97883 introduced a compile-time error in the experimental remote offloading
libomptarget plugin, this patch fixes it and resolves a number of
inconsistencies in the plugin as well:

1. Non-functional Asynchronous API
2. Unnecessarily verbose debug printing
3. Misc. code clean ups

This is not intended to make any functional changes to the plugin.

[jdoerfert]

LG

openmp/libomptarget/plugins/remote/include/Utils.h

 //===----------------- Utils.h - Utilities for Remote RTL -----------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // Utilities for data transfer through protobuf and debugging.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef UTILS_H
 #define UTILS_H
 
 #include "Debug.h"
 #include "omptarget.h"
 #include "openmp.grpc.pb.h"
 #include "openmp.pb.h"
 #include "rtl.h"
 #include <string>
 
 #define CLIENT_DBG(...)                                                        \
   {                                                                            \

 
 using namespace openmp::libomptarget::remote;
 
 using openmp::libomptarget::remote::DeviceOffloadEntry;
 using openmp::libomptarget::remote::TargetBinaryDescription;
 using openmp::libomptarget::remote::TargetOffloadEntry;
 using openmp::libomptarget::remote::TargetTable;
 
-struct RPCConfig {
+struct ClientManagerConfigTy {
   std::vector<std::string> ServerAddresses;
   uint64_t MaxSize;
   uint64_t BlockSize;
-  RPCConfig() {
-    ServerAddresses = {"0.0.0.0:50051"};
-    MaxSize = 1 << 30;
-    BlockSize = 1 << 20;
+  int Timeout;
+
+  ClientManagerConfigTy()
+      : ServerAddresses({"0.0.0.0:50051"}), MaxSize(1 << 30),
+        BlockSize(1 << 20), Timeout(5) {
+    // TODO: Error handle for incorrect inputs
+    if (const char *Env = std::getenv("LIBOMPTARGET_RPC_ADDRESS")) {
+      ServerAddresses.clear();
+      std::string AddressString = Env;
+      const std::string Delimiter = ",";
+
+      size_t Pos;
+      std::string Token;
+      while ((Pos = AddressString.find(Delimiter)) != std::string::npos) {
+        Token = AddressString.substr(0, Pos);
+        ServerAddresses.push_back(Token);
+        AddressString.erase(0, Pos + Delimiter.length());
+      }
+      ServerAddresses.push_back(AddressString);
+    }
+    if (const char *Env = std::getenv("LIBOMPTARGET_RPC_ALLOCATOR_MAX"))
+      MaxSize = std::stoi(Env);
+    if (const char *Env = std::getenv("LIBOMPTARGET_RPC_BLOCK_SIZE"))
+      BlockSize = std::stoi(Env);
+    if (const char *Env1 = std::getenv("LIBOMPTARGET_RPC_LATENCY"))
+      Timeout = std::stoi(Env1);
   }
 };
 
-/// Helper function to parse common environment variables between client/server
-void parseEnvironment(RPCConfig &Config);
-
 /// Loads a target binary description into protobuf.
 void loadTargetBinaryDescription(const __tgt_bin_desc *Desc,
                                  TargetBinaryDescription &Request);
 
 /// Unload a target binary description from protobuf. The map is used to keep
 /// track of already copied device images.
 void unloadTargetBinaryDescription(
     const TargetBinaryDescription *Request, __tgt_bin_desc *Desc,

openmp/libomptarget/plugins/remote/include/openmp.proto

service RemoteOffload {
 
   rpc IsValidBinary(TargetDeviceImagePtr) returns (I32) {}
   rpc GetNumberOfDevices(Null) returns (I32) {}
 
   rpc InitDevice(I32) returns (I32) {}
   rpc InitRequires(I64) returns (I32) {}
 
   rpc LoadBinary(Binary) returns (TargetTable) {}
-  rpc Synchronize(SynchronizeDevice) returns (I32) {}
 
   rpc DataAlloc(AllocData) returns (Pointer) {}
   rpc DataDelete(DeleteData) returns (I32) {}
 
-  rpc DataSubmitAsync(stream SubmitDataAsync) returns (I32) {}
-  rpc DataRetrieveAsync(RetrieveDataAsync) returns (stream Data) {}
+  rpc DataSubmit(stream SubmitData) returns (I32) {}
+  rpc DataRetrieve(RetrieveData) returns (stream Data) {}
 
   rpc IsDataExchangeable(DevicePair) returns (I32) {}
-  rpc DataExchangeAsync(ExchangeDataAsync) returns (I32) {}
+  rpc DataExchange(ExchangeData) returns (I32) {}
 
-  rpc RunTargetRegionAsync(TargetRegionAsync) returns (I32) {}
-  rpc RunTargetTeamRegionAsync(TargetTeamRegionAsync) returns (I32) {}
+  rpc RunTargetRegion(TargetRegion) returns (I32) {}
+  rpc RunTargetTeamRegion(TargetTeamRegion) returns (I32) {}
 }
 
 message Null {}
 
 message Pointer { uint64 number = 1; }
 
 message I32 { int32 number = 1; }
 

 message TargetBinaryDescription {
   repeated ImagePtrs image_ptrs = 1;
   repeated TargetOffloadEntry entries = 2;
   repeated TargetDeviceImage images = 3;
   repeated uint64 entry_ptrs = 4;
   uint64 bin_ptr = 5;
 }
 
-message SynchronizeDevice {
-  uint64 queue_ptr = 1;
-  int32 device_id = 2;
-}
-
 message AllocData {
   uint64 size = 1;
   uint64 hst_ptr = 2;
   int32 device_id = 3;
 }
 
-message SubmitDataAsync {
+message SubmitData {
   bytes data = 1;
   uint64 hst_ptr = 2;
   uint64 tgt_ptr = 3;
-  uint64 queue_ptr = 4;
   uint64 start = 5;
   uint64 size = 6;
   int32 device_id = 7;
 }
 
-message RetrieveDataAsync {
+message RetrieveData {
   uint64 hst_ptr = 1;
   uint64 tgt_ptr = 2;
   uint64 size = 3;
-  uint64 queue_ptr = 4;
   int32 device_id = 5;
 }
 
 message Data {
   bytes data = 1;
   uint64 start = 2;
   uint64 size = 3;
   int32 ret = 4;
 }
 
-message ExchangeDataAsync {
+message ExchangeData {
   uint64 src_dev_id = 1;
   uint64 src_ptr = 2;
   uint64 dst_dev_id = 3;
   uint64 dst_ptr = 4;
-  uint64 queue_ptr = 5;
   uint64 size = 6;
 }
 
 message DeleteData {
   uint64 tgt_ptr = 1;
   int32 device_id = 2;
 }
 
-message TargetRegionAsync {
+message TargetRegion {
   repeated uint64 tgt_args = 1;
   repeated int64 tgt_offsets = 2;
   uint64 tgt_entry_ptr = 3;
-  uint64 queue_ptr = 4;
-  int32 device_id = 5;
-  int32 arg_num = 6;
+  int32 device_id = 4;
+  int32 arg_num = 5;
 }
 
-message TargetTeamRegionAsync {
+message TargetTeamRegion {
   repeated uint64 tgt_args = 1;
   repeated int64 tgt_offsets = 2;
   uint64 tgt_entry_ptr = 3;
   uint64 loop_tripcount = 4;
-  uint64 queue_ptr = 5;
-  int32 device_id = 6;
-  int32 arg_num = 7;
-  int32 team_num = 8;
-  int32 thread_limit = 9;
+  int32 device_id = 5;
+  int32 arg_num = 6;
+  int32 team_num = 7;
+  int32 thread_limit = 8;
 }

openmp/libomptarget/plugins/remote/lib/Utils.cpp

 //===---------------- Utils.cpp - Utilities for Remote RTL ----------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // Utilities for data movement and debugging.
 //
 //===----------------------------------------------------------------------===//
 
 #include "Utils.h"
 #include "omptarget.h"
 
 namespace RemoteOffloading {
-void parseEnvironment(RPCConfig &Config) {
-  // TODO: Error handle for incorrect inputs
-  if (const char *Env = std::getenv("LIBOMPTARGET_RPC_ADDRESS")) {
-    Config.ServerAddresses.clear();
-    std::string AddressString = Env;
-    const std::string Delimiter = ",";
-
-    size_t Pos = 0;
-    std::string Token;
-    while ((Pos = AddressString.find(Delimiter)) != std::string::npos) {
-      Token = AddressString.substr(0, Pos);
-      Config.ServerAddresses.push_back(Token);
-      AddressString.erase(0, Pos + Delimiter.length());
-    }
-    Config.ServerAddresses.push_back(AddressString);
-  }
-  if (const char *Env = std::getenv("LIBOMPTARGET_RPC_ALLOCATOR_MAX"))
-    Config.MaxSize = std::stoi(Env);
-  if (const char *Env = std::getenv("LIBOMPTARGET_RPC_BLOCK_SIZE"))
-    Config.BlockSize = std::stoi(Env);
-}
 
 void loadTargetBinaryDescription(const __tgt_bin_desc *Desc,
                                  TargetBinaryDescription &Request) {
   // Keeps track of entries which have already been deep copied.
   std::vector<void *> DeepCopiedEntryAddrs;
 
   // Copy Global Offload Entries
   for (auto *CurEntry = Desc->HostEntriesBegin;
if (Request->entries_size())
     Desc->HostEntriesBegin = new __tgt_offload_entry[Request->entries_size()];
   else {
     Desc->HostEntriesBegin = nullptr;
     Desc->HostEntriesEnd = nullptr;
   }
 
   // Copy Global Offload Entries
   __tgt_offload_entry *CurEntry = Desc->HostEntriesBegin;
-  for (int i = 0; i < Request->entries_size(); i++) {
-    copyOffloadEntry(Request->entries()[i], CurEntry);
-    CopiedOffloadEntries[(void *)Request->entry_ptrs()[i]] = CurEntry;
+  size_t I = 0;
+  for (auto &Entry : Request->entries()) {
+    copyOffloadEntry(Entry, CurEntry);
+    CopiedOffloadEntries[(void *)Request->entry_ptrs()[I]] = CurEntry;
     CurEntry++;
+    I++;
   }
   Desc->HostEntriesEnd = CurEntry;
 
   // Copy Device Images and Device Offload Entries
   __tgt_device_image *CurImage = Desc->DeviceImages;
   auto ImageItr = Request->image_ptrs().begin();
   for (auto Image : Request->images()) {
     // Copy Device Offload Entries
-    auto *CurEntry = Desc->HostEntriesBegin;
+    CurEntry = Desc->HostEntriesBegin;
     bool Found = false;
 
     if (!Desc->HostEntriesBegin) {
       CurImage->EntriesBegin = nullptr;
       CurImage->EntriesEnd = nullptr;
     }
 
-    for (int i = 0; i < Image.entries_size(); i++) {
+    for (size_t I = 0; I < Image.entries_size(); I++) {
       auto TgtEntry =
-          CopiedOffloadEntries.find((void *)Request->entry_ptrs()[i]);
+          CopiedOffloadEntries.find((void *)Request->entry_ptrs()[I]);
       if (TgtEntry != CopiedOffloadEntries.end()) {
         if (!Found)
           CurImage->EntriesBegin = CurEntry;
 
-        Found = true;
-        if (Found) {
-          CurImage->EntriesEnd = CurEntry + 1;
-        }
+        Found = true;
       } else {
         Found = false;
-        copyOffloadEntry(Image.entries()[i], CurEntry);
-        CopiedOffloadEntries[(void *)(Request->entry_ptrs()[i])] = CurEntry;
+        copyOffloadEntry(Image.entries()[I], CurEntry);
+        CopiedOffloadEntries[(void *)(Request->entry_ptrs()[I])] = CurEntry;
       }
       CurEntry++;
     }
 
     // Copy Device Image
     CurImage->ImageStart = new uint8_t[Image.binary().size()];
     memcpy(CurImage->ImageStart,
            static_cast<const void *>(Image.binary().data()),

 }
 
 void unloadTargetTable(
     TargetTable &TableResponse, __tgt_target_table *Table,
     std::unordered_map<void *, void *> &HostToRemoteTargetTableMap) {
   Table->EntriesBegin = new __tgt_offload_entry[TableResponse.entries_size()];
 
   auto *CurEntry = Table->EntriesBegin;
-  for (int i = 0; i < TableResponse.entries_size(); i++) {
-    copyOffloadEntry(TableResponse.entries()[i], CurEntry);
+  for (size_t I = 0; I < TableResponse.entries_size(); I++) {
+    copyOffloadEntry(TableResponse.entries()[I], CurEntry);
     HostToRemoteTargetTableMap[CurEntry->addr] =
-        (void *)TableResponse.entry_ptrs()[i];
+        (void *)TableResponse.entry_ptrs()[I];
     CurEntry++;
   }
   Table->EntriesEnd = CurEntry;
 }
 
 void copyOffloadEntry(const TargetOffloadEntry &EntryResponse,
                       __tgt_offload_entry *Entry) {
   Entry->name = strdup(EntryResponse.name().c_str());

 void dump(__tgt_target_table *Table) {
   for (auto *CurEntry = Table->EntriesBegin; CurEntry != Table->EntriesEnd;
        CurEntry++)
     dump(CurEntry);
 }
 
 void dump(TargetOffloadEntry Entry) {
   fprintf(stderr, "Entry: ");
-  fprintf(stderr, "    %s\n", Entry.name().c_str());
-  fprintf(stderr, "    %d\n", Entry.reserved());
-  fprintf(stderr, "    %d\n", Entry.flags());
-  fprintf(stderr, "    %ld\n", Entry.data().size());
+  fprintf(stderr, "  Name: %s\n", Entry.name().c_str());
+  fprintf(stderr, "  Reserved: %d\n", Entry.reserved());
+  fprintf(stderr, "  Flags: %d\n", Entry.flags());
+  fprintf(stderr, "  Size:  %ld\n", Entry.data().size());
   dump(static_cast<const void *>(Entry.data().data()),
        static_cast<const void *>((Entry.data().c_str() + Entry.data().size())));
 }
 
 void dump(__tgt_device_image *Image) {
   dump(Image->ImageStart, Image->ImageEnd);
   __tgt_offload_entry *EntryItr = Image->EntriesBegin;
   for (; EntryItr != Image->EntriesEnd; EntryItr++)
     dump(EntryItr);
 }
 
 void dump(std::unordered_map<void *, __tgt_offload_entry *> &Map) {
   fprintf(stderr, "Host to Remote Entry Map:\n");
   for (auto Entry : Map)
     fprintf(stderr, "  Host (%p) -> Tgt (%p): Addr((%p))\n", Entry.first,
             (void *)Entry.second, (void *)Entry.second->addr);
 }
 } // namespace RemoteOffloading
 No newline at end of file

openmp/libomptarget/plugins/remote/server/OffloadingServer.cpp

 //===------------- OffloadingServer.cpp - Server Application --------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // Offloading server for remote host.
 //
 //===----------------------------------------------------------------------===//
 
 #include <future>
 #include <grpcpp/server.h>
 #include <grpcpp/server_builder.h>
 #include <iostream>
 #include <thread>
 
 #include "Server.h"
 
 using grpc::Server;
 using grpc::ServerBuilder;
 
 std::promise<void> ShutdownPromise;
 
 int main() {
-  RPCConfig Config;
-  parseEnvironment(Config);
+  ClientManagerConfigTy Config;
 
   RemoteOffloadImpl Service(Config.MaxSize, Config.BlockSize);
 
   ServerBuilder Builder;
   Builder.AddListeningPort(Config.ServerAddresses[0],
                            grpc::InsecureServerCredentials());
   Builder.RegisterService(&Service);
   Builder.SetMaxMessageSize(INT_MAX);

openmp/libomptarget/plugins/remote/server/Server.h

 //===-------------------------- Server.h - Server -------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // Offloading gRPC server for remote host.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_OPENMP_LIBOMPTARGET_PLUGINS_REMOTE_SERVER_SERVER_H
 #define LLVM_OPENMP_LIBOMPTARGET_PLUGINS_REMOTE_SERVER_SERVER_H
 
 #include <grpcpp/server_context.h>
 
 #include "Utils.h"
 #include "device.h"
 #include "omptarget.h"
 #include "openmp.grpc.pb.h"
 #include "openmp.pb.h"
 #include "rtl.h"
 

 extern PluginManager *PM;
 
 class RemoteOffloadImpl final : public RemoteOffload::Service {
 private:
   int32_t mapHostRTLDeviceId(int32_t RTLDeviceID);
 
   std::unordered_map<const void *, __tgt_device_image *>
       HostToRemoteDeviceImage;
-  std::unordered_map<const void *, __tgt_offload_entry *>
-      HostToRemoteOffloadEntry;
   std::unordered_map<const void *, std::unique_ptr<__tgt_bin_desc>>
       Descriptions;
   __tgt_target_table *Table = nullptr;
 
   int DebugLevel;
   uint64_t MaxSize;
   uint64_t BlockSize;
   std::unique_ptr<protobuf::Arena> Arena;
public:
 
   Status InitDevice(ServerContext *Context, const I32 *DeviceNum,
                     I32 *Reply) override;
   Status InitRequires(ServerContext *Context, const I64 *RequiresFlag,
                       I32 *Reply) override;
 
   Status LoadBinary(ServerContext *Context, const Binary *Binary,
                     TargetTable *Reply) override;
-  Status Synchronize(ServerContext *Context, const SynchronizeDevice *Info,
-                     I32 *Reply) override;
   Status IsDataExchangeable(ServerContext *Context, const DevicePair *Request,
                             I32 *Reply) override;
 
   Status DataAlloc(ServerContext *Context, const AllocData *Request,
                    Pointer *Reply) override;
 
-  Status DataSubmitAsync(ServerContext *Context,
-                         ServerReader<SubmitDataAsync> *Reader,
-                         I32 *Reply) override;
-  Status DataRetrieveAsync(ServerContext *Context,
-                           const RetrieveDataAsync *Request,
-                           ServerWriter<Data> *Writer) override;
+  Status DataSubmit(ServerContext *Context, ServerReader<SubmitData> *Reader,
+  Status DataRetrieve(ServerContext *Context, const RetrieveData *Request,
 
-  Status DataExchangeAsync(ServerContext *Context,
-                           const ExchangeDataAsync *Request,
-                           I32 *Reply) override;
+  Status DataExchange(ServerContext *Context, const ExchangeData *Request,
 
   Status DataDelete(ServerContext *Context, const DeleteData *Request,
                     I32 *Reply) override;
 
-  Status RunTargetRegionAsync(ServerContext *Context,
-                              const TargetRegionAsync *Request,
-                              I32 *Reply) override;
+  Status RunTargetRegion(ServerContext *Context, const TargetRegion *Request,
 
-  Status RunTargetTeamRegionAsync(ServerContext *Context,
-                                  const TargetTeamRegionAsync *Request,
-                                  I32 *Reply) override;
+  Status RunTargetTeamRegion(ServerContext *Context,
+                             const TargetTeamRegion *Request,
 };
 
 #endif

openmp/libomptarget/plugins/remote/server/Server.cpp

 #include "openmp.pb.h"
 
 using grpc::WriteOptions;
 
 extern std::promise<void> ShutdownPromise;
 
 Status RemoteOffloadImpl::Shutdown(ServerContext *Context, const Null *Request,
                                    I32 *Reply) {
-  SERVER_DBG("Shutting down the server");
+  SERVER_DBG("Shutting down the server")
 
   Reply->set_number(0);
   ShutdownPromise.set_value();
   return Status::OK;
 }
 
 Status
 RemoteOffloadImpl::RegisterLib(ServerContext *Context,
                                const TargetBinaryDescription *Description,
                                I32 *Reply) {
-  SERVER_DBG("Registering library");
-
   auto Desc = std::make_unique<__tgt_bin_desc>();
 
   unloadTargetBinaryDescription(Description, Desc.get(),
                                 HostToRemoteDeviceImage);
   PM->RTLs.RegisterLib(Desc.get());
 
   if (Descriptions.find((void *)Description->bin_ptr()) != Descriptions.end())
     freeTargetBinaryDescription(
         Descriptions[(void *)Description->bin_ptr()].get());
   else
     Descriptions[(void *)Description->bin_ptr()] = std::move(Desc);
 
-  SERVER_DBG("Registered library");
+  SERVER_DBG("Registered library")
   Reply->set_number(0);
   return Status::OK;
 }
 
 Status RemoteOffloadImpl::UnregisterLib(ServerContext *Context,
                                         const Pointer *Request, I32 *Reply) {
-  SERVER_DBG("Unregistering library");
-
   if (Descriptions.find((void *)Request->number()) == Descriptions.end()) {
     Reply->set_number(1);
     return Status::OK;
   }
 
   PM->RTLs.UnregisterLib(Descriptions[(void *)Request->number()].get());
   freeTargetBinaryDescription(Descriptions[(void *)Request->number()].get());
   Descriptions.erase((void *)Request->number());
 
-  SERVER_DBG("Unregistered library");
+  SERVER_DBG("Unregistered library")
   Reply->set_number(0);
   return Status::OK;
 }
 
 Status RemoteOffloadImpl::IsValidBinary(ServerContext *Context,
                                         const TargetDeviceImagePtr *DeviceImage,
                                         I32 *IsValid) {
-  SERVER_DBG("Checking if binary (%p) is valid",
-             (void *)(DeviceImage->image_ptr()));
-
   __tgt_device_image *Image =
       HostToRemoteDeviceImage[(void *)DeviceImage->image_ptr()];
 
   IsValid->set_number(0);
 
   for (auto &RTL : PM->RTLs.AllRTLs)
     if (auto Ret = RTL.is_valid_binary(Image)) {
       IsValid->set_number(Ret);
       break;
     }
 
   SERVER_DBG("Checked if binary (%p) is valid",
-             (void *)(DeviceImage->image_ptr()));
+             (void *)(DeviceImage->image_ptr()))
   return Status::OK;
 }
 
 Status RemoteOffloadImpl::GetNumberOfDevices(ServerContext *Context,
                                              const Null *Null,
                                              I32 *NumberOfDevices) {
-  SERVER_DBG("Getting number of devices");
   std::call_once(PM->RTLs.initFlag, &RTLsTy::LoadRTLs, &PM->RTLs);
 
   int32_t Devices = 0;
   PM->RTLsMtx.lock();
   for (auto &RTL : PM->RTLs.AllRTLs)
     Devices += RTL.NumberOfDevices;
   PM->RTLsMtx.unlock();
 
   NumberOfDevices->set_number(Devices);
 
-  SERVER_DBG("Got number of devices");
+  SERVER_DBG("Got number of devices")
   return Status::OK;
 }
 
 Status RemoteOffloadImpl::InitDevice(ServerContext *Context,
                                      const I32 *DeviceNum, I32 *Reply) {
-  SERVER_DBG("Initializing device %d", DeviceNum->number());
-
   Reply->set_number(PM->Devices[DeviceNum->number()].RTL->init_device(
       mapHostRTLDeviceId(DeviceNum->number())));
 
-  SERVER_DBG("Initialized device %d", DeviceNum->number());
+  SERVER_DBG("Initialized device %d", DeviceNum->number())
   return Status::OK;
 }
 
 Status RemoteOffloadImpl::InitRequires(ServerContext *Context,
                                        const I64 *RequiresFlag, I32 *Reply) {
-  SERVER_DBG("Initializing requires for devices");
-
   for (auto &Device : PM->Devices)
     if (Device.RTL->init_requires)
       Device.RTL->init_requires(RequiresFlag->number());
   Reply->set_number(RequiresFlag->number());
 
-  SERVER_DBG("Initialized requires for devices");
+  SERVER_DBG("Initialized requires for devices")
   return Status::OK;
 }
 
 Status RemoteOffloadImpl::LoadBinary(ServerContext *Context,
                                      const Binary *Binary, TargetTable *Reply) {
-  SERVER_DBG("Loading binary (%p) to device %d", (void *)Binary->image_ptr(),
-             Binary->device_id());
-
   __tgt_device_image *Image =
       HostToRemoteDeviceImage[(void *)Binary->image_ptr()];
 
   Table = PM->Devices[Binary->device_id()].RTL->load_binary(
       mapHostRTLDeviceId(Binary->device_id()), Image);
   if (Table)
     loadTargetTable(Table, *Reply, Image);
 
   SERVER_DBG("Loaded binary (%p) to device %d", (void *)Binary->image_ptr(),
-             Binary->device_id());
-  return Status::OK;
-}
-
-Status RemoteOffloadImpl::Synchronize(ServerContext *Context,
-                                      const SynchronizeDevice *Info,
-                                      I32 *Reply) {
-  SERVER_DBG("Synchronizing device %d (probably won't work)",
-             Info->device_id());
-
-  void *AsyncInfo = (void *)Info->queue_ptr();
-  Reply->set_number(0);
-  if (PM->Devices[Info->device_id()].RTL->synchronize)
-    Reply->set_number(PM->Devices[Info->device_id()].synchronize(
-        (__tgt_async_info *)AsyncInfo));
-
-  SERVER_DBG("Synchronized device %d", Info->device_id());
+             Binary->device_id())
   return Status::OK;
 }
 
 Status RemoteOffloadImpl::IsDataExchangeable(ServerContext *Context,
                                              const DevicePair *Request,
                                              I32 *Reply) {
-  SERVER_DBG("Checking if data exchangable between device %d and device %d",
-             Request->src_dev_id(), Request->dst_dev_id());
-
   Reply->set_number(-1);
   if (PM->Devices[mapHostRTLDeviceId(Request->src_dev_id())]
           .RTL->is_data_exchangable)
     Reply->set_number(PM->Devices[mapHostRTLDeviceId(Request->src_dev_id())]
                           .RTL->is_data_exchangable(Request->src_dev_id(),
                                                     Request->dst_dev_id()));
 
-  SERVER_DBG("Checked if data exchangable between device %d and device %d",
-             Request->src_dev_id(), Request->dst_dev_id());
+  SERVER_DBG("Checked if data exchangeable between device %d and device %d",
+             Request->src_dev_id(), Request->dst_dev_id())
   return Status::OK;
 }
 
 Status RemoteOffloadImpl::DataAlloc(ServerContext *Context,
                                     const AllocData *Request, Pointer *Reply) {
-  SERVER_DBG("Allocating %ld bytes on sevice %d", Request->size(),
-             Request->device_id());
-
   uint64_t TgtPtr = (uint64_t)PM->Devices[Request->device_id()].RTL->data_alloc(
       mapHostRTLDeviceId(Request->device_id()), Request->size(),
-      (void *)Request->hst_ptr());
+      (void *)Request->hst_ptr(), TARGET_ALLOC_DEFAULT);
   Reply->set_number(TgtPtr);
 
-  SERVER_DBG("Allocated at " DPxMOD "", DPxPTR((void *)TgtPtr));
+  SERVER_DBG("Allocated at " DPxMOD "", DPxPTR((void *)TgtPtr))
 
   return Status::OK;
 }
 
-Status RemoteOffloadImpl::DataSubmitAsync(ServerContext *Context,
-                                          ServerReader<SubmitDataAsync> *Reader,
-                                          I32 *Reply) {
-  SubmitDataAsync Request;
+Status RemoteOffloadImpl::DataSubmit(ServerContext *Context,
+                                     ServerReader<SubmitData> *Reader,
+  SubmitData Request;
   uint8_t *HostCopy = nullptr;
   while (Reader->Read(&Request)) {
     if (Request.start() == 0 && Request.size() == Request.data().size()) {
-      SERVER_DBG("Submitting %lu bytes async to (%p) on device %d",
-                 Request.data().size(), (void *)Request.tgt_ptr(),
-                 Request.device_id());
-
-      SERVER_DBG("  Host Pointer Info: %p, %p", (void *)Request.hst_ptr(),
-                 static_cast<const void *>(Request.data().data()));
-
       Reader->SendInitialMetadata();
 
       Reply->set_number(PM->Devices[Request.device_id()].RTL->data_submit(
           mapHostRTLDeviceId(Request.device_id()), (void *)Request.tgt_ptr(),
           (void *)Request.data().data(), Request.data().size()));
 
       SERVER_DBG("Submitted %lu bytes async to (%p) on device %d",
                  Request.data().size(), (void *)Request.tgt_ptr(),
-                 Request.device_id());
+                 Request.device_id())
 
       return Status::OK;
     }
     if (!HostCopy) {
       HostCopy = new uint8_t[Request.size()];
       Reader->SendInitialMetadata();
     }
 
-    SERVER_DBG("Submitting %lu-%lu/%lu bytes async to (%p) on device %d",
-               Request.start(), Request.start() + Request.data().size(),
-               Request.size(), (void *)Request.tgt_ptr(), Request.device_id());
-
     memcpy((void *)((char *)HostCopy + Request.start()), Request.data().data(),
            Request.data().size());
   }
-  SERVER_DBG("  Host Pointer Info: %p, %p", (void *)Request.hst_ptr(),
-             static_cast<const void *>(Request.data().data()));
 
   Reply->set_number(PM->Devices[Request.device_id()].RTL->data_submit(
       mapHostRTLDeviceId(Request.device_id()), (void *)Request.tgt_ptr(),
       HostCopy, Request.size()));
 
   delete[] HostCopy;
 
   SERVER_DBG("Submitted %lu bytes to (%p) on device %d", Request.data().size(),
-             (void *)Request.tgt_ptr(), Request.device_id());
+             (void *)Request.tgt_ptr(), Request.device_id())
 
   return Status::OK;
 }
 
-Status RemoteOffloadImpl::DataRetrieveAsync(ServerContext *Context,
-                                            const RetrieveDataAsync *Request,
-                                            ServerWriter<Data> *Writer) {
+Status RemoteOffloadImpl::DataRetrieve(ServerContext *Context,
+                                       const RetrieveData *Request,
   auto HstPtr = std::make_unique<char[]>(Request->size());
+
   auto Ret = PM->Devices[Request->device_id()].RTL->data_retrieve(
       mapHostRTLDeviceId(Request->device_id()), HstPtr.get(),
       (void *)Request->tgt_ptr(), Request->size());
 
   if (Arena->SpaceAllocated() >= MaxSize)
     Arena->Reset();
 
   if (Request->size() > BlockSize) {
     uint64_t Start = 0, End = BlockSize;
     for (auto I = 0; I < ceil((float)Request->size() / BlockSize); I++) {
       auto *Reply = protobuf::Arena::CreateMessage<Data>(Arena.get());
 
       Reply->set_start(Start);
       Reply->set_size(Request->size());
       Reply->set_data((char *)HstPtr.get() + Start, End - Start);
       Reply->set_ret(Ret);
 
-      SERVER_DBG("Retrieving %lu-%lu/%lu bytes from (%p) on device %d", Start,
-                 End, Request->size(), (void *)Request->tgt_ptr(),
-                 mapHostRTLDeviceId(Request->device_id()));
-
       if (!Writer->Write(*Reply)) {
-        CLIENT_DBG("Broken stream when submitting data");
+        CLIENT_DBG("Broken stream when submitting data")
       }
 
       SERVER_DBG("Retrieved %lu-%lu/%lu bytes from (%p) on device %d", Start,
                  End, Request->size(), (void *)Request->tgt_ptr(),
-                 mapHostRTLDeviceId(Request->device_id()));
+                 mapHostRTLDeviceId(Request->device_id()))
 
       Start += BlockSize;
       End += BlockSize;
       if (End >= Request->size())
         End = Request->size();
     }
   } else {
     auto *Reply = protobuf::Arena::CreateMessage<Data>(Arena.get());
 
-    SERVER_DBG("Retrieve %lu bytes from (%p) on device %d", Request->size(),
-               (void *)Request->tgt_ptr(),
-               mapHostRTLDeviceId(Request->device_id()));
-
     Reply->set_start(0);
     Reply->set_size(Request->size());
     Reply->set_data((char *)HstPtr.get(), Request->size());
     Reply->set_ret(Ret);
 
     SERVER_DBG("Retrieved %lu bytes from (%p) on device %d", Request->size(),
                (void *)Request->tgt_ptr(),
-               mapHostRTLDeviceId(Request->device_id()));
+               mapHostRTLDeviceId(Request->device_id()))
 
     Writer->WriteLast(*Reply, WriteOptions());
   }
 
   return Status::OK;
 }
 
-Status RemoteOffloadImpl::DataExchangeAsync(ServerContext *Context,
-                                            const ExchangeDataAsync *Request,
-                                            I32 *Reply) {
-  SERVER_DBG(
-      "Exchanging data asynchronously from device %d (%p) to device %d (%p) of "
-      "size %lu",
-      mapHostRTLDeviceId(Request->src_dev_id()), (void *)Request->src_ptr(),
-      mapHostRTLDeviceId(Request->dst_dev_id()), (void *)Request->dst_ptr(),
-      Request->size());
-
+Status RemoteOffloadImpl::DataExchange(ServerContext *Context,
+                                       const ExchangeData *Request,
   if (PM->Devices[Request->src_dev_id()].RTL->data_exchange) {
     int32_t Ret = PM->Devices[Request->src_dev_id()].RTL->data_exchange(
         mapHostRTLDeviceId(Request->src_dev_id()), (void *)Request->src_ptr(),
         mapHostRTLDeviceId(Request->dst_dev_id()), (void *)Request->dst_ptr(),
         Request->size());
     Reply->set_number(Ret);
   } else
     Reply->set_number(-1);
 
   SERVER_DBG(
       "Exchanged data asynchronously from device %d (%p) to device %d (%p) of "
       "size %lu",
       mapHostRTLDeviceId(Request->src_dev_id()), (void *)Request->src_ptr(),
       mapHostRTLDeviceId(Request->dst_dev_id()), (void *)Request->dst_ptr(),
-      Request->size());
+      Request->size())
   return Status::OK;
 }
 
 Status RemoteOffloadImpl::DataDelete(ServerContext *Context,
                                      const DeleteData *Request, I32 *Reply) {
-  SERVER_DBG("Deleting data from (%p) on device %d", (void *)Request->tgt_ptr(),
-             mapHostRTLDeviceId(Request->device_id()));
-
   auto Ret = PM->Devices[Request->device_id()].RTL->data_delete(
       mapHostRTLDeviceId(Request->device_id()), (void *)Request->tgt_ptr());
   Reply->set_number(Ret);
 
   SERVER_DBG("Deleted data from (%p) on device %d", (void *)Request->tgt_ptr(),
-             mapHostRTLDeviceId(Request->device_id()));
+             mapHostRTLDeviceId(Request->device_id()))
   return Status::OK;
 }
 
-Status RemoteOffloadImpl::RunTargetRegionAsync(ServerContext *Context,
-                                               const TargetRegionAsync *Request,
-                                               I32 *Reply) {
-  SERVER_DBG("Running TargetRegionAsync on device %d with %d args",
-             mapHostRTLDeviceId(Request->device_id()), Request->arg_num());
-
+Status RemoteOffloadImpl::RunTargetRegion(ServerContext *Context,
+                                          const TargetRegion *Request,
   std::vector<uint8_t> TgtArgs(Request->arg_num());
   for (auto I = 0; I < Request->arg_num(); I++)
     TgtArgs[I] = (uint64_t)Request->tgt_args()[I];
 
   std::vector<ptrdiff_t> TgtOffsets(Request->arg_num());
   const auto *TgtOffsetItr = Request->tgt_offsets().begin();
   for (auto I = 0; I < Request->arg_num(); I++, TgtOffsetItr++)
     TgtOffsets[I] = (ptrdiff_t)*TgtOffsetItr;
 
   void *TgtEntryPtr = ((__tgt_offload_entry *)Request->tgt_entry_ptr())->addr;
 
   int32_t Ret = PM->Devices[Request->device_id()].RTL->run_region(
       mapHostRTLDeviceId(Request->device_id()), TgtEntryPtr,
       (void **)TgtArgs.data(), TgtOffsets.data(), Request->arg_num());
 
   Reply->set_number(Ret);
 
-  SERVER_DBG("Ran TargetRegionAsync on device %d with %d args",
-             mapHostRTLDeviceId(Request->device_id()), Request->arg_num());
+  SERVER_DBG("Ran TargetRegion on device %d with %d args",
+             mapHostRTLDeviceId(Request->device_id()), Request->arg_num())
   return Status::OK;
 }
 
-Status RemoteOffloadImpl::RunTargetTeamRegionAsync(
-    ServerContext *Context, const TargetTeamRegionAsync *Request, I32 *Reply) {
-  SERVER_DBG("Running TargetTeamRegionAsync on device %d with %d args",
-             mapHostRTLDeviceId(Request->device_id()), Request->arg_num());
-
+Status RemoteOffloadImpl::RunTargetTeamRegion(ServerContext *Context,
+                                              const TargetTeamRegion *Request,
+                                              I32 *Reply) {
   std::vector<uint64_t> TgtArgs(Request->arg_num());
   for (auto I = 0; I < Request->arg_num(); I++)
     TgtArgs[I] = (uint64_t)Request->tgt_args()[I];
 
   std::vector<ptrdiff_t> TgtOffsets(Request->arg_num());
   const auto *TgtOffsetItr = Request->tgt_offsets().begin();
   for (auto I = 0; I < Request->arg_num(); I++, TgtOffsetItr++)
     TgtOffsets[I] = (ptrdiff_t)*TgtOffsetItr;
 
   void *TgtEntryPtr = ((__tgt_offload_entry *)Request->tgt_entry_ptr())->addr;
+
   int32_t Ret = PM->Devices[Request->device_id()].RTL->run_team_region(
       mapHostRTLDeviceId(Request->device_id()), TgtEntryPtr,
       (void **)TgtArgs.data(), TgtOffsets.data(), Request->arg_num(),
       Request->team_num(), Request->thread_limit(), Request->loop_tripcount());
 
   Reply->set_number(Ret);
 
-  SERVER_DBG("Ran TargetTeamRegionAsync on device %d with %d args",
-             mapHostRTLDeviceId(Request->device_id()), Request->arg_num());
+  SERVER_DBG("Ran TargetTeamRegion on device %d with %d args",
+             mapHostRTLDeviceId(Request->device_id()), Request->arg_num())
   return Status::OK;
 }
 
 int32_t RemoteOffloadImpl::mapHostRTLDeviceId(int32_t RTLDeviceID) {
   for (auto &RTL : PM->RTLs.UsedRTLs) {
     if (RTLDeviceID - RTL->NumberOfDevices >= 0)
       RTLDeviceID -= RTL->NumberOfDevices;
     else
       break;
   }
   return RTLDeviceID;
 }

openmp/libomptarget/plugins/remote/src/Client.h

 //===------------------ Client.h - Client Implementation ------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // gRPC Client for the remote plugin.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_OPENMP_LIBOMPTARGET_PLUGINS_REMOTE_SRC_CLIENT_H
 #define LLVM_OPENMP_LIBOMPTARGET_PLUGINS_REMOTE_SRC_CLIENT_H
 
 #include "Utils.h"
 #include "omptarget.h"
 #include <google/protobuf/arena.h>
 #include <grpcpp/grpcpp.h>
 #include <grpcpp/security/credentials.h>
 #include <grpcpp/support/channel_arguments.h>
 #include <memory>
 #include <mutex>
 #include <numeric>
 
 using grpc::Channel;
 using openmp::libomptarget::remote::RemoteOffload;
 using namespace RemoteOffloading;
 
 using namespace google;
 
 class RemoteOffloadClient {
-  const int Timeout;
-
   int DebugLevel;
-  uint64_t MaxSize;
-  int64_t BlockSize;
+  const int Timeout;
+  const uint64_t MaxSize;
+  const int64_t BlockSize;
 
   std::unique_ptr<RemoteOffload::Stub> Stub;
   std::unique_ptr<protobuf::Arena> Arena;
 
   std::unique_ptr<std::mutex> ArenaAllocatorLock;
 
   std::map<int32_t, std::unordered_map<void *, void *>> RemoteEntries;
   std::map<int32_t, std::unique_ptr<__tgt_target_table>> DevicesToTables;
 
   template <typename Fn1, typename Fn2, typename TReturn>
-  auto remoteCall(Fn1 Preprocess, Fn2 Postprocess, TReturn ErrorValue,
-                  bool Timeout = true);
+  auto remoteCall(Fn1 Preprocessor, Fn2 Postprocessor, TReturn ErrorValue,
+                  bool CanTimeOut = true);
 
 public:
   RemoteOffloadClient(std::shared_ptr<Channel> Channel, int Timeout,
                       uint64_t MaxSize, int64_t BlockSize)
       : Timeout(Timeout), MaxSize(MaxSize), BlockSize(BlockSize),
         Stub(RemoteOffload::NewStub(Channel)) {
     DebugLevel = getDebugLevel();
     Arena = std::make_unique<protobuf::Arena>();
public:
 
   int32_t isValidBinary(__tgt_device_image *Image);
   int32_t getNumberOfDevices();
 
   int32_t initDevice(int32_t DeviceId);
   int32_t initRequires(int64_t RequiresFlags);
 
   __tgt_target_table *loadBinary(int32_t DeviceId, __tgt_device_image *Image);
-  int64_t synchronize(int32_t DeviceId, __tgt_async_info *AsyncInfo);
-  int32_t isDataExchangeable(int32_t SrcDevId, int32_t DstDevId);
 
   void *dataAlloc(int32_t DeviceId, int64_t Size, void *HstPtr);
   int32_t dataDelete(int32_t DeviceId, void *TgtPtr);
 
-  int32_t dataSubmitAsync(int32_t DeviceId, void *TgtPtr, void *HstPtr,
-                          int64_t Size, __tgt_async_info *AsyncInfo);
-  int32_t dataRetrieveAsync(int32_t DeviceId, void *HstPtr, void *TgtPtr,
-                            int64_t Size, __tgt_async_info *AsyncInfo);
-
-  int32_t dataExchangeAsync(int32_t SrcDevId, void *SrcPtr, int32_t DstDevId,
-                            void *DstPtr, int64_t Size,
-                            __tgt_async_info *AsyncInfo);
+  int32_t dataSubmit(int32_t DeviceId, void *TgtPtr, void *HstPtr,
+                     int64_t Size);
+  int32_t dataRetrieve(int32_t DeviceId, void *HstPtr, void *TgtPtr,
+                       int64_t Size);
 
-  int32_t runTargetRegionAsync(int32_t DeviceId, void *TgtEntryPtr,
-                               void **TgtArgs, ptrdiff_t *TgtOffsets,
-                               int32_t ArgNum, __tgt_async_info *AsyncInfo);
+  int32_t isDataExchangeable(int32_t SrcDevId, int32_t DstDevId);
+  int32_t dataExchange(int32_t SrcDevId, void *SrcPtr, int32_t DstDevId,
+                       void *DstPtr, int64_t Size);
 
-  int32_t runTargetTeamRegionAsync(int32_t DeviceId, void *TgtEntryPtr,
-                                   void **TgtArgs, ptrdiff_t *TgtOffsets,
-                                   int32_t ArgNum, int32_t TeamNum,
-                                   int32_t ThreadLimit, uint64_t LoopTripCount,
-                                   __tgt_async_info *AsyncInfo);
+  int32_t runTargetRegion(int32_t DeviceId, void *TgtEntryPtr, void **TgtArgs,
+                          ptrdiff_t *TgtOffsets, int32_t ArgNum);
+  int32_t runTargetTeamRegion(int32_t DeviceId, void *TgtEntryPtr,
+                              int32_t ThreadLimit, uint64_t LoopTripCount);
 };
 
 class RemoteClientManager {
 private:
-  std::vector<std::string> Addresses;
   std::vector<RemoteOffloadClient> Clients;
   std::vector<int> Devices;
 
   std::pair<int32_t, int32_t> mapDeviceId(int32_t DeviceId);
   int DebugLevel;
 
 public:
-  RemoteClientManager(std::vector<std::string> Addresses, int Timeout,
-                      uint64_t MaxSize, int64_t BlockSize)
-      : Addresses(Addresses) {
+  RemoteClientManager() {
+    ClientManagerConfigTy Config;
+
     grpc::ChannelArguments ChArgs;
     ChArgs.SetMaxReceiveMessageSize(-1);
     DebugLevel = getDebugLevel();
-    for (auto Address : Addresses) {
+    for (auto Address : Config.ServerAddresses) {
       Clients.push_back(RemoteOffloadClient(
           grpc::CreateChannel(Address, grpc::InsecureChannelCredentials()),
-          Timeout, MaxSize, BlockSize));
+          Config.Timeout, Config.MaxSize, Config.BlockSize));
     }
   }
 
   int32_t shutdown(void);
 
   int32_t registerLib(__tgt_bin_desc *Desc);
   int32_t unregisterLib(__tgt_bin_desc *Desc);
 
   int32_t isValidBinary(__tgt_device_image *Image);
   int32_t getNumberOfDevices();
 
   int32_t initDevice(int32_t DeviceId);
   int32_t initRequires(int64_t RequiresFlags);
 
   __tgt_target_table *loadBinary(int32_t DeviceId, __tgt_device_image *Image);
-  int64_t synchronize(int32_t DeviceId, __tgt_async_info *AsyncInfo);
-  int32_t isDataExchangeable(int32_t SrcDevId, int32_t DstDevId);
 
   void *dataAlloc(int32_t DeviceId, int64_t Size, void *HstPtr);
   int32_t dataDelete(int32_t DeviceId, void *TgtPtr);
 
-  int32_t dataSubmitAsync(int32_t DeviceId, void *TgtPtr, void *HstPtr,
-                          int64_t Size, __tgt_async_info *AsyncInfo);
-  int32_t dataRetrieveAsync(int32_t DeviceId, void *HstPtr, void *TgtPtr,
-                            int64_t Size, __tgt_async_info *AsyncInfo);
-
-  int32_t dataExchangeAsync(int32_t SrcDevId, void *SrcPtr, int32_t DstDevId,
-                            void *DstPtr, int64_t Size,
-                            __tgt_async_info *AsyncInfo);
+  int32_t dataSubmit(int32_t DeviceId, void *TgtPtr, void *HstPtr,
+                     int64_t Size);
+  int32_t dataRetrieve(int32_t DeviceId, void *HstPtr, void *TgtPtr,
+                       int64_t Size);
 
-  int32_t runTargetRegionAsync(int32_t DeviceId, void *TgtEntryPtr,
-                               void **TgtArgs, ptrdiff_t *TgtOffsets,
-                               int32_t ArgNum, __tgt_async_info *AsyncInfo);
+  int32_t isDataExchangeable(int32_t SrcDevId, int32_t DstDevId);
+  int32_t dataExchange(int32_t SrcDevId, void *SrcPtr, int32_t DstDevId,
+                       void *DstPtr, int64_t Size);
 
-  int32_t runTargetTeamRegionAsync(int32_t DeviceId, void *TgtEntryPtr,
-                                   void **TgtArgs, ptrdiff_t *TgtOffsets,
-                                   int32_t ArgNum, int32_t TeamNum,
-                                   int32_t ThreadLimit, uint64_t LoopTripCount,
-                                   __tgt_async_info *AsyncInfo);
+  int32_t runTargetRegion(int32_t DeviceId, void *TgtEntryPtr, void **TgtArgs,
+                          ptrdiff_t *TgtOffsets, int32_t ArgNum);
+  int32_t runTargetTeamRegion(int32_t DeviceId, void *TgtEntryPtr,
+                              int32_t ThreadLimit, uint64_t LoopTripCount);
 };
 
 #endif

openmp/libomptarget/plugins/remote/src/Client.cpp

 using namespace std::chrono;
 
 using grpc::ClientContext;
 using grpc::ClientReader;
 using grpc::ClientWriter;
 using grpc::Status;
 
 template <typename Fn1, typename Fn2, typename TReturn>
-auto RemoteOffloadClient::remoteCall(Fn1 Preprocess, Fn2 Postprocess,
-                                     TReturn ErrorValue, bool Timeout) {
+auto RemoteOffloadClient::remoteCall(Fn1 Preprocessor, Fn2 Postprocessor,
+                                     TReturn ErrorValue, bool CanTimeOut) {
   ArenaAllocatorLock->lock();
   if (Arena->SpaceAllocated() >= MaxSize)
     Arena->Reset();
   ArenaAllocatorLock->unlock();
 
   ClientContext Context;
-  if (Timeout) {
+  if (CanTimeOut) {
     auto Deadline =
         std::chrono::system_clock::now() + std::chrono::seconds(Timeout);
     Context.set_deadline(Deadline);
   }
 
   Status RPCStatus;
-  auto Reply = Preprocess(RPCStatus, Context);
+  auto Reply = Preprocessor(RPCStatus, Context);
 
-  // TODO: Error handle more appropriately
   if (!RPCStatus.ok()) {
-    CLIENT_DBG("%s", RPCStatus.error_message().c_str());
+    CLIENT_DBG("%s", RPCStatus.error_message().c_str())
   } else {
-    return Postprocess(Reply);
+    return Postprocessor(Reply);
   }
 
-  CLIENT_DBG("Failed");
+  CLIENT_DBG("Failed")
   return ErrorValue;
 }
 
 int32_t RemoteOffloadClient::shutdown(void) {
   ClientContext Context;
   Null Request;
   I32 Reply;
-  CLIENT_DBG("Shutting down server.");
+  CLIENT_DBG("Shutting down server.")
   auto Status = Stub->Shutdown(&Context, Request, &Reply);
   if (Status.ok())
     return Reply.number();
   return 1;
 }
 
 int32_t RemoteOffloadClient::registerLib(__tgt_bin_desc *Desc) {
   return remoteCall(
-      /* Preprocess */
+      /* Preprocessor */
       [&](auto &RPCStatus, auto &Context) {
         auto *Request = protobuf::Arena::CreateMessage<TargetBinaryDescription>(
             Arena.get());
         auto *Reply = protobuf::Arena::CreateMessage<I32>(Arena.get());
         loadTargetBinaryDescription(Desc, *Request);
         Request->set_bin_ptr((uint64_t)Desc);
 
-        CLIENT_DBG("Registering library");
         RPCStatus = Stub->RegisterLib(&Context, *Request, Reply);
         return Reply;
       },
-      /* Postprocess */
+      /* Postprocessor */
       [&](const auto &Reply) {
         if (Reply->number() == 0) {
-          CLIENT_DBG("Registered library");
+          CLIENT_DBG("Registered library")
           return 0;
         }
         return 1;
       },
       /* Error Value */ 1);
 }
 
 int32_t RemoteOffloadClient::unregisterLib(__tgt_bin_desc *Desc) {
   return remoteCall(
-      /* Preprocess */
+      /* Preprocessor */
       [&](auto &RPCStatus, auto &Context) {
         auto *Request = protobuf::Arena::CreateMessage<Pointer>(Arena.get());
         auto *Reply = protobuf::Arena::CreateMessage<I32>(Arena.get());
 
         Request->set_number((uint64_t)Desc);
 
-        CLIENT_DBG("Unregistering library");
         RPCStatus = Stub->UnregisterLib(&Context, *Request, Reply);
         return Reply;
       },
-      /* Postprocess */
+      /* Postprocessor */
       [&](const auto &Reply) {
         if (Reply->number() == 0) {
-          CLIENT_DBG("Unregistered library");
+          CLIENT_DBG("Unregistered library")
           return 0;
         }
-        CLIENT_DBG("Failed to unregister library");
+        CLIENT_DBG("Failed to unregister library")
         return 1;
       },
       /* Error Value */ 1);
 }
 
 int32_t RemoteOffloadClient::isValidBinary(__tgt_device_image *Image) {
   return remoteCall(
-      /* Preprocess */
+      /* Preprocessor */
       [&](auto &RPCStatus, auto &Context) {
         auto *Request =
             protobuf::Arena::CreateMessage<TargetDeviceImagePtr>(Arena.get());
         auto *Reply = protobuf::Arena::CreateMessage<I32>(Arena.get());
 
         Request->set_image_ptr((uint64_t)Image->ImageStart);
 
         auto *EntryItr = Image->EntriesBegin;
         while (EntryItr != Image->EntriesEnd)
           Request->add_entry_ptrs((uint64_t)EntryItr++);
 
-        CLIENT_DBG("Validating binary");
         RPCStatus = Stub->IsValidBinary(&Context, *Request, Reply);
         return Reply;
       },
-      /* Postprocess */
+      /* Postprocessor */
       [&](const auto &Reply) {
         if (Reply->number()) {
-          CLIENT_DBG("Validated binary");
+          CLIENT_DBG("Validated binary")
         } else {
-          CLIENT_DBG("Could not validate binary");
+          CLIENT_DBG("Could not validate binary")
         }
         return Reply->number();
       },
       /* Error Value */ 0);
 }
 
 int32_t RemoteOffloadClient::getNumberOfDevices() {
   return remoteCall(
-      /* Preprocess */
+      /* Preprocessor */
       [&](Status &RPCStatus, ClientContext &Context) {
         auto *Request = protobuf::Arena::CreateMessage<Null>(Arena.get());
         auto *Reply = protobuf::Arena::CreateMessage<I32>(Arena.get());
 
-        CLIENT_DBG("Getting number of devices");
         RPCStatus = Stub->GetNumberOfDevices(&Context, *Request, Reply);
 
         return Reply;
       },
-      /* Postprocess */
+      /* Postprocessor */
       [&](const auto &Reply) {
         if (Reply->number()) {
-          CLIENT_DBG("Found %d devices", Reply->number());
+          CLIENT_DBG("Found %d devices", Reply->number())
         } else {
-          CLIENT_DBG("Could not get the number of devices");
+          CLIENT_DBG("Could not get the number of devices")
         }
         return Reply->number();
       },
       /*Error Value*/ -1);
 }
 
 int32_t RemoteOffloadClient::initDevice(int32_t DeviceId) {
   return remoteCall(
-      /* Preprocess */
+      /* Preprocessor */
       [&](auto &RPCStatus, auto &Context) {
         auto *Request = protobuf::Arena::CreateMessage<I32>(Arena.get());
         auto *Reply = protobuf::Arena::CreateMessage<I32>(Arena.get());
 
         Request->set_number(DeviceId);
 
-        CLIENT_DBG("Initializing device %d", DeviceId);
         RPCStatus = Stub->InitDevice(&Context, *Request, Reply);
 
         return Reply;
       },
-      /* Postprocess */
+      /* Postprocessor */
       [&](const auto &Reply) {
         if (!Reply->number()) {
-          CLIENT_DBG("Initialized device %d", DeviceId);
+          CLIENT_DBG("Initialized device %d", DeviceId)
         } else {
-          CLIENT_DBG("Could not initialize device %d", DeviceId);
+          CLIENT_DBG("Could not initialize device %d", DeviceId)
         }
         return Reply->number();
       },
       /* Error Value */ -1);
 }
 
 int32_t RemoteOffloadClient::initRequires(int64_t RequiresFlags) {
   return remoteCall(
-      /* Preprocess */
+      /* Preprocessor */
       [&](auto &RPCStatus, auto &Context) {
         auto *Request = protobuf::Arena::CreateMessage<I64>(Arena.get());
         auto *Reply = protobuf::Arena::CreateMessage<I32>(Arena.get());
         Request->set_number(RequiresFlags);
-        CLIENT_DBG("Initializing requires");
         RPCStatus = Stub->InitRequires(&Context, *Request, Reply);
         return Reply;
       },
-      /* Postprocess */
+      /* Postprocessor */
       [&](const auto &Reply) {
         if (Reply->number()) {
-          CLIENT_DBG("Initialized requires");
+          CLIENT_DBG("Initialized requires")
         } else {
-          CLIENT_DBG("Could not initialize requires");
+          CLIENT_DBG("Could not initialize requires")
         }
         return Reply->number();
       },
       /* Error Value */ -1);
 }
 
 __tgt_target_table *RemoteOffloadClient::loadBinary(int32_t DeviceId,
                                                     __tgt_device_image *Image) {
   return remoteCall(
-      /* Preprocess */
+      /* Preprocessor */
       [&](auto &RPCStatus, auto &Context) {
         auto *ImageMessage =
             protobuf::Arena::CreateMessage<Binary>(Arena.get());
         auto *Reply = protobuf::Arena::CreateMessage<TargetTable>(Arena.get());
         ImageMessage->set_image_ptr((uint64_t)Image->ImageStart);
         ImageMessage->set_device_id(DeviceId);
 
-        CLIENT_DBG("Loading Image %p to device %d", Image, DeviceId);
         RPCStatus = Stub->LoadBinary(&Context, *ImageMessage, Reply);
         return Reply;
       },
-      /* Postprocess */
+      /* Postprocessor */
       [&](auto &Reply) {
         if (Reply->entries_size() == 0) {
-          CLIENT_DBG("Could not load image %p onto device %d", Image, DeviceId);
+          CLIENT_DBG("Could not load image %p onto device %d", Image, DeviceId)
           return (__tgt_target_table *)nullptr;
         }
         DevicesToTables[DeviceId] = std::make_unique<__tgt_target_table>();
         unloadTargetTable(*Reply, DevicesToTables[DeviceId].get(),
                           RemoteEntries[DeviceId]);
 
         CLIENT_DBG("Loaded Image %p to device %d with %d entries", Image,
-                   DeviceId, Reply->entries_size());
+                   DeviceId, Reply->entries_size())
 
         return DevicesToTables[DeviceId].get();
       },
       /* Error Value */ (__tgt_target_table *)nullptr,
-      /* Timeout */ false);
-}
-
-int64_t RemoteOffloadClient::synchronize(int32_t DeviceId,
-                                         __tgt_async_info *AsyncInfo) {
-  return remoteCall(
-      /* Preprocess */
-      [&](auto &RPCStatus, auto &Context) {
-        auto *Reply = protobuf::Arena::CreateMessage<I32>(Arena.get());
-        auto *Info =
-            protobuf::Arena::CreateMessage<SynchronizeDevice>(Arena.get());
-
-        Info->set_device_id(DeviceId);
-        Info->set_queue_ptr((uint64_t)AsyncInfo);
-
-        CLIENT_DBG("Synchronizing device %d", DeviceId);
-        RPCStatus = Stub->Synchronize(&Context, *Info, Reply);
-        return Reply;
-      },
-      /* Postprocess */
-      [&](auto &Reply) {
-        if (Reply->number()) {
-          CLIENT_DBG("Synchronized device %d", DeviceId);
-        } else {
-          CLIENT_DBG("Could not synchronize device %d", DeviceId);
-        }
-        return Reply->number();
-      },
-      /* Error Value */ -1);
+      /* CanTimeOut */ false);
 }
 
 int32_t RemoteOffloadClient::isDataExchangeable(int32_t SrcDevId,
                                                 int32_t DstDevId) {
   return remoteCall(
-      /* Preprocess */
+      /* Preprocessor */
       [&](auto &RPCStatus, auto &Context) {
         auto *Request = protobuf::Arena::CreateMessage<DevicePair>(Arena.get());
         auto *Reply = protobuf::Arena::CreateMessage<I32>(Arena.get());
 
         Request->set_src_dev_id(SrcDevId);
         Request->set_dst_dev_id(DstDevId);
 
-        CLIENT_DBG("Asking if data is exchangeable between %d, %d", SrcDevId,
-                   DstDevId);
         RPCStatus = Stub->IsDataExchangeable(&Context, *Request, Reply);
         return Reply;
       },
-      /* Postprocess */
+      /* Postprocessor */
       [&](auto &Reply) {
         if (Reply->number()) {
-          CLIENT_DBG("Data is exchangeable between %d, %d", SrcDevId, DstDevId);
+          CLIENT_DBG("Data is exchangeable between %d, %d", SrcDevId, DstDevId)
         } else {
           CLIENT_DBG("Data is not exchangeable between %d, %d", SrcDevId,
-                     DstDevId);
+                     DstDevId)
         }
         return Reply->number();
       },
       /* Error Value */ -1);
 }
 
 void *RemoteOffloadClient::dataAlloc(int32_t DeviceId, int64_t Size,
                                      void *HstPtr) {
   return remoteCall(
-      /* Preprocess */
+      /* Preprocessor */
       [&](auto &RPCStatus, auto &Context) {
         auto *Reply = protobuf::Arena::CreateMessage<Pointer>(Arena.get());
         auto *Request = protobuf::Arena::CreateMessage<AllocData>(Arena.get());
 
         Request->set_device_id(DeviceId);
         Request->set_size(Size);
         Request->set_hst_ptr((uint64_t)HstPtr);
 
-        CLIENT_DBG("Allocating %ld bytes on device %d", Size, DeviceId);
         RPCStatus = Stub->DataAlloc(&Context, *Request, Reply);
         return Reply;
       },
-      /* Postprocess */
+      /* Postprocessor */
       [&](auto &Reply) {
         if (Reply->number()) {
           CLIENT_DBG("Allocated %ld bytes on device %d at %p", Size, DeviceId,
-                     (void *)Reply->number());
+                     (void *)Reply->number())
         } else {
           CLIENT_DBG("Could not allocate %ld bytes on device %d at %p", Size,
-                     DeviceId, (void *)Reply->number());
+                     DeviceId, (void *)Reply->number())
         }
         return (void *)Reply->number();
       },
       /* Error Value */ (void *)nullptr);
 }
 
-int32_t RemoteOffloadClient::dataSubmitAsync(int32_t DeviceId, void *TgtPtr,
-                                             void *HstPtr, int64_t Size,
-                                             __tgt_async_info *AsyncInfo) {
+int32_t RemoteOffloadClient::dataSubmit(int32_t DeviceId, void *TgtPtr,
+                                        void *HstPtr, int64_t Size) {
 
   return remoteCall(
-      /* Preprocess */
+      /* Preprocessor */
       [&](auto &RPCStatus, auto &Context) {
         auto *Reply = protobuf::Arena::CreateMessage<I32>(Arena.get());
-        std::unique_ptr<ClientWriter<SubmitDataAsync>> Writer(
-            Stub->DataSubmitAsync(&Context, Reply));
+        std::unique_ptr<ClientWriter<SubmitData>> Writer(
+            Stub->DataSubmit(&Context, Reply));
 
         if (Size > BlockSize) {
           int64_t Start = 0, End = BlockSize;
           for (auto I = 0; I < ceil((float)Size / BlockSize); I++) {
             auto *Request =
-                protobuf::Arena::CreateMessage<SubmitDataAsync>(Arena.get());
+                protobuf::Arena::CreateMessage<SubmitData>(Arena.get());
 
             Request->set_device_id(DeviceId);
             Request->set_data((char *)HstPtr + Start, End - Start);
             Request->set_hst_ptr((uint64_t)HstPtr);
             Request->set_tgt_ptr((uint64_t)TgtPtr);
             Request->set_start(Start);
             Request->set_size(Size);
-            Request->set_queue_ptr((uint64_t)AsyncInfo);
-
-            CLIENT_DBG("Submitting %ld-%ld/%ld bytes async on device %d at %p",
-                       Start, End, Size, DeviceId, TgtPtr)
 
             if (!Writer->Write(*Request)) {
-              CLIENT_DBG("Broken stream when submitting data");
+              CLIENT_DBG("Broken stream when submitting data")
               Reply->set_number(0);
               return Reply;
             }
 
             Start += BlockSize;
             End += BlockSize;
             if (End >= Size)
               End = Size;
           }
         } else {
           auto *Request =
-              protobuf::Arena::CreateMessage<SubmitDataAsync>(Arena.get());
+              protobuf::Arena::CreateMessage<SubmitData>(Arena.get());
 
           Request->set_device_id(DeviceId);
           Request->set_data(HstPtr, Size);
           Request->set_hst_ptr((uint64_t)HstPtr);
           Request->set_tgt_ptr((uint64_t)TgtPtr);
           Request->set_start(0);
           Request->set_size(Size);
 
-          CLIENT_DBG("Submitting %ld bytes async on device %d at %p", Size,
-                     DeviceId, TgtPtr)
           if (!Writer->Write(*Request)) {
-            CLIENT_DBG("Broken stream when submitting data");
+            CLIENT_DBG("Broken stream when submitting data")
             Reply->set_number(0);
             return Reply;
           }
         }
 
         Writer->WritesDone();
         RPCStatus = Writer->Finish();
 
         return Reply;
       },
-      /* Postprocess */
+      /* Postprocessor */
       [&](auto &Reply) {
         if (!Reply->number()) {
-          CLIENT_DBG("Async submitted %ld bytes on device %d at %p", Size,
-                     DeviceId, TgtPtr)
+          CLIENT_DBG(" submitted %ld bytes on device %d at %p", Size, DeviceId,
+                     TgtPtr)
         } else {
           CLIENT_DBG("Could not async submit %ld bytes on device %d at %p",
                      Size, DeviceId, TgtPtr)
         }
         return Reply->number();
       },
       /* Error Value */ -1,
-      /* Timeout */ false);
+      /* CanTimeOut */ false);
 }
 
-int32_t RemoteOffloadClient::dataRetrieveAsync(int32_t DeviceId, void *HstPtr,
-                                               void *TgtPtr, int64_t Size,
-                                               __tgt_async_info *AsyncInfo) {
+int32_t RemoteOffloadClient::dataRetrieve(int32_t DeviceId, void *HstPtr,
+                                          void *TgtPtr, int64_t Size) {
   return remoteCall(
-      /* Preprocess */
+      /* Preprocessor */
       [&](auto &RPCStatus, auto &Context) {
         auto *Request =
-            protobuf::Arena::CreateMessage<RetrieveDataAsync>(Arena.get());
+            protobuf::Arena::CreateMessage<RetrieveData>(Arena.get());
 
         Request->set_device_id(DeviceId);
         Request->set_size(Size);
         Request->set_hst_ptr((int64_t)HstPtr);
         Request->set_tgt_ptr((int64_t)TgtPtr);
-        Request->set_queue_ptr((uint64_t)AsyncInfo);
 
         auto *Reply = protobuf::Arena::CreateMessage<Data>(Arena.get());
         std::unique_ptr<ClientReader<Data>> Reader(
-            Stub->DataRetrieveAsync(&Context, *Request));
+            Stub->DataRetrieve(&Context, *Request));
         Reader->WaitForInitialMetadata();
         while (Reader->Read(Reply)) {
           if (Reply->ret()) {
             CLIENT_DBG("Could not async retrieve %ld bytes on device %d at %p "
                        "for %p",
                        Size, DeviceId, TgtPtr, HstPtr)
             return Reply;
           }
 
           if (Reply->start() == 0 && Reply->size() == Reply->data().size()) {
-            CLIENT_DBG("Async retrieving %ld bytes on device %d at %p for %p",
-                       Size, DeviceId, TgtPtr, HstPtr)
-
             memcpy(HstPtr, Reply->data().data(), Reply->data().size());
 
             return Reply;
           }
-          CLIENT_DBG("Retrieving %lu-%lu/%lu bytes async from (%p) to (%p) "
-                     "on Device %d",
-                     Reply->start(), Reply->start() + Reply->data().size(),
-                     Reply->size(), (void *)Request->tgt_ptr(), HstPtr,
-                     Request->device_id());
 
           memcpy((void *)((char *)HstPtr + Reply->start()),
                  Reply->data().data(), Reply->data().size());
         }
         RPCStatus = Reader->Finish();
 
         return Reply;
       },
-      /* Postprocess */
+      /* Postprocessor */
       [&](auto &Reply) {
         if (!Reply->ret()) {
-          CLIENT_DBG("Async retrieve %ld bytes on Device %d", Size, DeviceId);
+          CLIENT_DBG("Retrieved %ld bytes on Device %d", Size, DeviceId)
         } else {
           CLIENT_DBG("Could not async retrieve %ld bytes on Device %d", Size,
-                     DeviceId);
+                     DeviceId)
         }
         return Reply->ret();
       },
       /* Error Value */ -1,
-      /* Timeout */ false);
+      /* CanTimeOut */ false);
 }
 
-int32_t RemoteOffloadClient::dataExchangeAsync(int32_t SrcDevId, void *SrcPtr,
-                                               int32_t DstDevId, void *DstPtr,
-                                               int64_t Size,
-                                               __tgt_async_info *AsyncInfo) {
+int32_t RemoteOffloadClient::dataExchange(int32_t SrcDevId, void *SrcPtr,
+                                          int64_t Size) {
   return remoteCall(
-      /* Preprocess */
+      /* Preprocessor */
       [&](auto &RPCStatus, auto &Context) {
         auto *Reply = protobuf::Arena::CreateMessage<I32>(Arena.get());
         auto *Request =
-            protobuf::Arena::CreateMessage<ExchangeDataAsync>(Arena.get());
+            protobuf::Arena::CreateMessage<ExchangeData>(Arena.get());
 
         Request->set_src_dev_id(SrcDevId);
         Request->set_src_ptr((uint64_t)SrcPtr);
         Request->set_dst_dev_id(DstDevId);
         Request->set_dst_ptr((uint64_t)DstPtr);
         Request->set_size(Size);
-        Request->set_queue_ptr((uint64_t)AsyncInfo);
 
-        CLIENT_DBG(
-            "Exchanging %ld bytes on device %d at %p for %p on device %d", Size,
-            SrcDevId, SrcPtr, DstPtr, DstDevId);
-        RPCStatus = Stub->DataExchangeAsync(&Context, *Request, Reply);
+        RPCStatus = Stub->DataExchange(&Context, *Request, Reply);
         return Reply;
       },
-      /* Postprocess */
+      /* Postprocessor */
       [&](auto &Reply) {
         if (Reply->number()) {
           CLIENT_DBG(
               "Exchanged %ld bytes on device %d at %p for %p on device %d",
-              Size, SrcDevId, SrcPtr, DstPtr, DstDevId);
+              Size, SrcDevId, SrcPtr, DstPtr, DstDevId)
         } else {
           CLIENT_DBG("Could not exchange %ld bytes on device %d at %p for %p "
                      "on device %d",
-                     Size, SrcDevId, SrcPtr, DstPtr, DstDevId);
+                     Size, SrcDevId, SrcPtr, DstPtr, DstDevId)
         }
         return Reply->number();
       },
       /* Error Value */ -1);
 }
 
 int32_t RemoteOffloadClient::dataDelete(int32_t DeviceId, void *TgtPtr) {
   return remoteCall(
-      /* Preprocess */
+      /* Preprocessor */
       [&](auto &RPCStatus, auto &Context) {
         auto *Reply = protobuf::Arena::CreateMessage<I32>(Arena.get());
         auto *Request = protobuf::Arena::CreateMessage<DeleteData>(Arena.get());
 
         Request->set_device_id(DeviceId);
         Request->set_tgt_ptr((uint64_t)TgtPtr);
 
-        CLIENT_DBG("Deleting data at %p on device %d", TgtPtr, DeviceId)
         RPCStatus = Stub->DataDelete(&Context, *Request, Reply);
         return Reply;
       },
-      /* Postprocess */
+      /* Postprocessor */
       [&](auto &Reply) {
         if (!Reply->number()) {
           CLIENT_DBG("Deleted data at %p on device %d", TgtPtr, DeviceId)
         } else {
           CLIENT_DBG("Could not delete data at %p on device %d", TgtPtr,
                      DeviceId)
         }
         return Reply->number();
       },
       /* Error Value */ -1);
 }
 
-int32_t RemoteOffloadClient::runTargetRegionAsync(
-    int32_t DeviceId, void *TgtEntryPtr, void **TgtArgs, ptrdiff_t *TgtOffsets,
-    int32_t ArgNum, __tgt_async_info *AsyncInfo) {
+int32_t RemoteOffloadClient::runTargetRegion(int32_t DeviceId,
+                                             void *TgtEntryPtr, void **TgtArgs,
+                                             ptrdiff_t *TgtOffsets,
+                                             int32_t ArgNum) {
   return remoteCall(
-      /* Preprocess */
+      /* Preprocessor */
       [&](auto &RPCStatus, auto &Context) {
         auto *Reply = protobuf::Arena::CreateMessage<I32>(Arena.get());
         auto *Request =
-            protobuf::Arena::CreateMessage<TargetRegionAsync>(Arena.get());
+            protobuf::Arena::CreateMessage<TargetRegion>(Arena.get());
 
         Request->set_device_id(DeviceId);
-        Request->set_queue_ptr((uint64_t)AsyncInfo);
 
         Request->set_tgt_entry_ptr(
             (uint64_t)RemoteEntries[DeviceId][TgtEntryPtr]);
 
         char **ArgPtr = (char **)TgtArgs;
         for (auto I = 0; I < ArgNum; I++, ArgPtr++)
           Request->add_tgt_args((uint64_t)*ArgPtr);
 
         char *OffsetPtr = (char *)TgtOffsets;
         for (auto I = 0; I < ArgNum; I++, OffsetPtr++)
           Request->add_tgt_offsets((uint64_t)*OffsetPtr);
 
         Request->set_arg_num(ArgNum);
 
-        CLIENT_DBG("Running target region async on device %d", DeviceId);
-        RPCStatus = Stub->RunTargetRegionAsync(&Context, *Request, Reply);
+        RPCStatus = Stub->RunTargetRegion(&Context, *Request, Reply);
         return Reply;
       },
-      /* Postprocess */
+      /* Postprocessor */
       [&](auto &Reply) {
         if (!Reply->number()) {
-          CLIENT_DBG("Ran target region async on device %d", DeviceId);
+          CLIENT_DBG("Ran target region async on device %d", DeviceId)
         } else {
-          CLIENT_DBG("Could not run target region async on device %d",
-                     DeviceId);
+          CLIENT_DBG("Could not run target region async on device %d", DeviceId)
         }
         return Reply->number();
       },
       /* Error Value */ -1,
-      /* Timeout */ false);
+      /* CanTimeOut */ false);
 }
 
-int32_t RemoteOffloadClient::runTargetTeamRegionAsync(
+int32_t RemoteOffloadClient::runTargetTeamRegion(
     int32_t DeviceId, void *TgtEntryPtr, void **TgtArgs, ptrdiff_t *TgtOffsets,
     int32_t ArgNum, int32_t TeamNum, int32_t ThreadLimit,
-    uint64_t LoopTripcount, __tgt_async_info *AsyncInfo) {
+    uint64_t LoopTripcount) {
   return remoteCall(
-      /* Preprocess */
+      /* Preprocessor */
       [&](auto &RPCStatus, auto &Context) {
         auto *Reply = protobuf::Arena::CreateMessage<I32>(Arena.get());
         auto *Request =
-            protobuf::Arena::CreateMessage<TargetTeamRegionAsync>(Arena.get());
+            protobuf::Arena::CreateMessage<TargetTeamRegion>(Arena.get());
 
         Request->set_device_id(DeviceId);
-        Request->set_queue_ptr((uint64_t)AsyncInfo);
 
         Request->set_tgt_entry_ptr(
             (uint64_t)RemoteEntries[DeviceId][TgtEntryPtr]);
 
         char **ArgPtr = (char **)TgtArgs;
         for (auto I = 0; I < ArgNum; I++, ArgPtr++) {
           Request->add_tgt_args((uint64_t)*ArgPtr);
         }
 
         char *OffsetPtr = (char *)TgtOffsets;
         for (auto I = 0; I < ArgNum; I++, OffsetPtr++)
           Request->add_tgt_offsets((uint64_t)*OffsetPtr);
 
         Request->set_arg_num(ArgNum);
         Request->set_team_num(TeamNum);
         Request->set_thread_limit(ThreadLimit);
         Request->set_loop_tripcount(LoopTripcount);
 
-        CLIENT_DBG("Running target team region async on device %d", DeviceId);
-        RPCStatus = Stub->RunTargetTeamRegionAsync(&Context, *Request, Reply);
+        RPCStatus = Stub->RunTargetTeamRegion(&Context, *Request, Reply);
         return Reply;
       },
-      /* Postprocess */
+      /* Postprocessor */
       [&](auto &Reply) {
         if (!Reply->number()) {
-          CLIENT_DBG("Ran target team region async on device %d", DeviceId);
+          CLIENT_DBG("Ran target team region async on device %d", DeviceId)
         } else {
           CLIENT_DBG("Could not run target team region async on device %d",
-                     DeviceId);
+                     DeviceId)
         }
         return Reply->number();
       },
       /* Error Value */ -1,
-      /* Timeout */ false);
+      /* CanTimeOut */ false);
 }
 
-// TODO: Better error handling for the next three functions
 int32_t RemoteClientManager::shutdown(void) {
   int32_t Ret = 0;
   for (auto &Client : Clients)
     Ret &= Client.shutdown();
   return Ret;
 }
 
 int32_t RemoteClientManager::registerLib(__tgt_bin_desc *Desc) {
for (auto &Client : Clients) {
     ClientIdx++;
   }
 
   return std::accumulate(Devices.begin(), Devices.end(), 0);
 }
 
 std::pair<int32_t, int32_t> RemoteClientManager::mapDeviceId(int32_t DeviceId) {
   for (size_t ClientIdx = 0; ClientIdx < Devices.size(); ClientIdx++) {
-    if (!(DeviceId >= Devices[ClientIdx]))
+    if (DeviceId < Devices[ClientIdx])
       return {ClientIdx, DeviceId};
     DeviceId -= Devices[ClientIdx];
   }
   return {-1, -1};
 }
 
 int32_t RemoteClientManager::initDevice(int32_t DeviceId) {
   int32_t ClientIdx, DeviceIdx;

 
 __tgt_target_table *RemoteClientManager::loadBinary(int32_t DeviceId,
                                                     __tgt_device_image *Image) {
   int32_t ClientIdx, DeviceIdx;
   std::tie(ClientIdx, DeviceIdx) = mapDeviceId(DeviceId);
   return Clients[ClientIdx].loadBinary(DeviceIdx, Image);
 }
 
-int64_t RemoteClientManager::synchronize(int32_t DeviceId,
-                                         __tgt_async_info *AsyncInfo) {
-  int32_t ClientIdx, DeviceIdx;
-  std::tie(ClientIdx, DeviceIdx) = mapDeviceId(DeviceId);
-  return Clients[ClientIdx].synchronize(DeviceIdx, AsyncInfo);
-}
-
 int32_t RemoteClientManager::isDataExchangeable(int32_t SrcDevId,
                                                 int32_t DstDevId) {
   int32_t SrcClientIdx, SrcDeviceIdx, DstClientIdx, DstDeviceIdx;
   std::tie(SrcClientIdx, SrcDeviceIdx) = mapDeviceId(SrcDevId);
   std::tie(DstClientIdx, DstDeviceIdx) = mapDeviceId(DstDevId);
   return Clients[SrcClientIdx].isDataExchangeable(SrcDeviceIdx, DstDeviceIdx);
 }
 
 void *RemoteClientManager::dataAlloc(int32_t DeviceId, int64_t Size,
                                      void *HstPtr) {
   int32_t ClientIdx, DeviceIdx;
   std::tie(ClientIdx, DeviceIdx) = mapDeviceId(DeviceId);
   return Clients[ClientIdx].dataAlloc(DeviceIdx, Size, HstPtr);
 }
 
 int32_t RemoteClientManager::dataDelete(int32_t DeviceId, void *TgtPtr) {
   int32_t ClientIdx, DeviceIdx;
   std::tie(ClientIdx, DeviceIdx) = mapDeviceId(DeviceId);
   return Clients[ClientIdx].dataDelete(DeviceIdx, TgtPtr);
 }
 
-int32_t RemoteClientManager::dataSubmitAsync(int32_t DeviceId, void *TgtPtr,
-                                             void *HstPtr, int64_t Size,
-                                             __tgt_async_info *AsyncInfo) {
+int32_t RemoteClientManager::dataSubmit(int32_t DeviceId, void *TgtPtr,
+                                        void *HstPtr, int64_t Size) {
   int32_t ClientIdx, DeviceIdx;
   std::tie(ClientIdx, DeviceIdx) = mapDeviceId(DeviceId);
-  return Clients[ClientIdx].dataSubmitAsync(DeviceIdx, TgtPtr, HstPtr, Size,
-                                            AsyncInfo);
+  return Clients[ClientIdx].dataSubmit(DeviceIdx, TgtPtr, HstPtr, Size);
 }
 
-int32_t RemoteClientManager::dataRetrieveAsync(int32_t DeviceId, void *HstPtr,
-                                               void *TgtPtr, int64_t Size,
-                                               __tgt_async_info *AsyncInfo) {
+int32_t RemoteClientManager::dataRetrieve(int32_t DeviceId, void *HstPtr,
+                                          void *TgtPtr, int64_t Size) {
   int32_t ClientIdx, DeviceIdx;
   std::tie(ClientIdx, DeviceIdx) = mapDeviceId(DeviceId);
-  return Clients[ClientIdx].dataRetrieveAsync(DeviceIdx, HstPtr, TgtPtr, Size,
-                                              AsyncInfo);
+  return Clients[ClientIdx].dataRetrieve(DeviceIdx, HstPtr, TgtPtr, Size);
 }
 
-int32_t RemoteClientManager::dataExchangeAsync(int32_t SrcDevId, void *SrcPtr,
-                                               int32_t DstDevId, void *DstPtr,
-                                               int64_t Size,
-                                               __tgt_async_info *AsyncInfo) {
+int32_t RemoteClientManager::dataExchange(int32_t SrcDevId, void *SrcPtr,
+                                          int64_t Size) {
   int32_t SrcClientIdx, SrcDeviceIdx, DstClientIdx, DstDeviceIdx;
   std::tie(SrcClientIdx, SrcDeviceIdx) = mapDeviceId(SrcDevId);
   std::tie(DstClientIdx, DstDeviceIdx) = mapDeviceId(DstDevId);
-  return Clients[SrcClientIdx].dataExchangeAsync(
-      SrcDeviceIdx, SrcPtr, DstDeviceIdx, DstPtr, Size, AsyncInfo);
+  return Clients[SrcClientIdx].dataExchange(SrcDeviceIdx, SrcPtr, DstDeviceIdx,
+                                            DstPtr, Size);
 }
 
-int32_t RemoteClientManager::runTargetRegionAsync(
-    int32_t DeviceId, void *TgtEntryPtr, void **TgtArgs, ptrdiff_t *TgtOffsets,
-    int32_t ArgNum, __tgt_async_info *AsyncInfo) {
+int32_t RemoteClientManager::runTargetRegion(int32_t DeviceId,
+                                             void *TgtEntryPtr, void **TgtArgs,
+                                             ptrdiff_t *TgtOffsets,
+                                             int32_t ArgNum) {
   int32_t ClientIdx, DeviceIdx;
   std::tie(ClientIdx, DeviceIdx) = mapDeviceId(DeviceId);
-  return Clients[ClientIdx].runTargetRegionAsync(
-      DeviceIdx, TgtEntryPtr, TgtArgs, TgtOffsets, ArgNum, AsyncInfo);
+  return Clients[ClientIdx].runTargetRegion(DeviceIdx, TgtEntryPtr, TgtArgs,
+                                            TgtOffsets, ArgNum);
 }
 
-int32_t RemoteClientManager::runTargetTeamRegionAsync(
+int32_t RemoteClientManager::runTargetTeamRegion(
     int32_t DeviceId, void *TgtEntryPtr, void **TgtArgs, ptrdiff_t *TgtOffsets,
     int32_t ArgNum, int32_t TeamNum, int32_t ThreadLimit,
-    uint64_t LoopTripCount, __tgt_async_info *AsyncInfo) {
+    uint64_t LoopTripCount) {
   int32_t ClientIdx, DeviceIdx;
   std::tie(ClientIdx, DeviceIdx) = mapDeviceId(DeviceId);
-  return Clients[ClientIdx].runTargetTeamRegionAsync(
-      DeviceIdx, TgtEntryPtr, TgtArgs, TgtOffsets, ArgNum, TeamNum, ThreadLimit,
-      LoopTripCount, AsyncInfo);
+  return Clients[ClientIdx].runTargetTeamRegion(DeviceIdx, TgtEntryPtr, TgtArgs,
+                                                TgtOffsets, ArgNum, TeamNum,
+                                                ThreadLimit, LoopTripCount);
 }

openmp/libomptarget/plugins/remote/src/rtl.cpp

 #define TARGET_NAME RPC
 #define DEBUG_PREFIX "Target " GETNAME(TARGET_NAME) " RTL"
 
 RemoteClientManager *Manager;
 
 __attribute__((constructor(101))) void initRPC() {
   DP("Init RPC library!\n");
 
-  RPCConfig Config;
-  parseEnvironment(Config);
-
-  int Timeout = 5;
-  if (const char *Env1 = std::getenv("LIBOMPTARGET_RPC_LATENCY"))
-    Timeout = std::stoi(Env1);
-
-  Manager = new RemoteClientManager(Config.ServerAddresses, Timeout,
-                                    Config.MaxSize, Config.BlockSize);
+  Manager = new RemoteClientManager();
 }
 
 __attribute__((destructor(101))) void deinitRPC() {
   Manager->shutdown(); // TODO: Error handle shutting down
   DP("Deinit RPC library!\n");
   delete Manager;
 }
 
int64_t __tgt_rtl_init_requires(int64_t RequiresFlags) {
   return Manager->initRequires(RequiresFlags);
 }
 
 __tgt_target_table *__tgt_rtl_load_binary(int32_t DeviceId,
                                           __tgt_device_image *Image) {
   return Manager->loadBinary(DeviceId, (__tgt_device_image *)Image);
 }
 
-int32_t __tgt_rtl_synchronize(int32_t DeviceId, __tgt_async_info *AsyncInfo) {
-  return Manager->synchronize(DeviceId, AsyncInfo);
-}
-
 int32_t __tgt_rtl_is_data_exchangable(int32_t SrcDevId, int32_t DstDevId) {
   return Manager->isDataExchangeable(SrcDevId, DstDevId);
 }
 
 void *__tgt_rtl_data_alloc(int32_t DeviceId, int64_t Size, void *HstPtr,
-                           int32_t kind) {
-  if (kind != TARGET_ALLOC_DEFAULT) {
+                           int32_t Kind) {
+  if (Kind != TARGET_ALLOC_DEFAULT) {
     REPORT("Invalid target data allocation kind or requested allocator not "
            "implemented yet\n");
     return NULL;
   }
 
   return Manager->dataAlloc(DeviceId, Size, HstPtr);
 }
 
 int32_t __tgt_rtl_data_submit(int32_t DeviceId, void *TgtPtr, void *HstPtr,
                               int64_t Size) {
-  return Manager->dataSubmitAsync(DeviceId, TgtPtr, HstPtr, Size, nullptr);
-}
-
-int32_t __tgt_rtl_data_submit_async(int32_t DeviceId, void *TgtPtr,
-                                    void *HstPtr, int64_t Size,
-                                    __tgt_async_info *AsyncInfo) {
-  return Manager->dataSubmitAsync(DeviceId, TgtPtr, HstPtr, Size, AsyncInfo);
+  return Manager->dataSubmit(DeviceId, TgtPtr, HstPtr, Size);
 }
 
 int32_t __tgt_rtl_data_retrieve(int32_t DeviceId, void *HstPtr, void *TgtPtr,
                                 int64_t Size) {
-  return Manager->dataRetrieveAsync(DeviceId, HstPtr, TgtPtr, Size, nullptr);
-}
-
-int32_t __tgt_rtl_data_retrieve_async(int32_t DeviceId, void *HstPtr,
-                                      void *TgtPtr, int64_t Size,
-                                      __tgt_async_info *AsyncInfo) {
-  return Manager->dataRetrieveAsync(DeviceId, HstPtr, TgtPtr, Size, AsyncInfo);
+  return Manager->dataRetrieve(DeviceId, HstPtr, TgtPtr, Size);
 }
 
 int32_t __tgt_rtl_data_delete(int32_t DeviceId, void *TgtPtr) {
   return Manager->dataDelete(DeviceId, TgtPtr);
 }
 
 int32_t __tgt_rtl_data_exchange(int32_t SrcDevId, void *SrcPtr,
                                 int32_t DstDevId, void *DstPtr, int64_t Size) {
-  return Manager->dataExchangeAsync(SrcDevId, SrcPtr, DstDevId, DstPtr, Size,
-                                    nullptr);
+  return Manager->dataExchange(SrcDevId, SrcPtr, DstDevId, DstPtr, Size);
 }
 
-int32_t __tgt_rtl_data_exchange_async(int32_t SrcDevId, void *SrcPtr,
-                                      int32_t DstDevId, void *DstPtr,
-                                      int64_t Size,
-                                      __tgt_async_info *AsyncInfo) {
-  return Manager->dataExchangeAsync(SrcDevId, SrcPtr, DstDevId, DstPtr, Size,
-                                    AsyncInfo);
-}
 
 int32_t __tgt_rtl_run_target_region(int32_t DeviceId, void *TgtEntryPtr,
                                     void **TgtArgs, ptrdiff_t *TgtOffsets,
                                     int32_t ArgNum) {
-  return Manager->runTargetRegionAsync(DeviceId, TgtEntryPtr, TgtArgs,
-                                       TgtOffsets, ArgNum, nullptr);
-}
-
-int32_t __tgt_rtl_run_target_region_async(int32_t DeviceId, void *TgtEntryPtr,
-                                          void **TgtArgs, ptrdiff_t *TgtOffsets,
-                                          int32_t ArgNum,
-                                          __tgt_async_info *AsyncInfo) {
-  return Manager->runTargetRegionAsync(DeviceId, TgtEntryPtr, TgtArgs,
-                                       TgtOffsets, ArgNum, AsyncInfo);
+  return Manager->runTargetRegion(DeviceId, TgtEntryPtr, TgtArgs, TgtOffsets,
+                                  ArgNum);
 }
 
 int32_t __tgt_rtl_run_target_team_region(int32_t DeviceId, void *TgtEntryPtr,
                                          void **TgtArgs, ptrdiff_t *TgtOffsets,
                                          int32_t ArgNum, int32_t TeamNum,
                                          int32_t ThreadLimit,
                                          uint64_t LoopTripCount) {
-  return Manager->runTargetTeamRegionAsync(DeviceId, TgtEntryPtr, TgtArgs,
-                                           TgtOffsets, ArgNum, TeamNum,
-                                           ThreadLimit, LoopTripCount, nullptr);
-}
-
-int32_t __tgt_rtl_run_target_team_region_async(
-    int32_t DeviceId, void *TgtEntryPtr, void **TgtArgs, ptrdiff_t *TgtOffsets,
-    int32_t ArgNum, int32_t TeamNum, int32_t ThreadLimit,
-    uint64_t LoopTripCount, __tgt_async_info *AsyncInfo) {
-  return Manager->runTargetTeamRegionAsync(
-      DeviceId, TgtEntryPtr, TgtArgs, TgtOffsets, ArgNum, TeamNum, ThreadLimit,
-      LoopTripCount, AsyncInfo);
+  return Manager->runTargetTeamRegion(DeviceId, TgtEntryPtr, TgtArgs,
+                                      TgtOffsets, ArgNum, TeamNum, ThreadLimit,
+                                      LoopTripCount);
 }
 
 // Exposed library API function
 #ifdef __cplusplus
 }
 #endif