Index: openmp/trunk/libomptarget/src/CMakeLists.txt =================================================================== --- openmp/trunk/libomptarget/src/CMakeLists.txt +++ openmp/trunk/libomptarget/src/CMakeLists.txt @@ -14,6 +14,7 @@ libomptarget_say("Building offloading runtime library libomptarget.") set(src_files + api.cpp device.cpp rtl.cpp omptarget.cpp Index: openmp/trunk/libomptarget/src/api.cpp =================================================================== --- openmp/trunk/libomptarget/src/api.cpp +++ openmp/trunk/libomptarget/src/api.cpp @@ -0,0 +1,282 @@ +//===----------- api.cpp - Target independent OpenMP target RTL -----------===// +// +// The LLVM Compiler Infrastructure +// +// This file is dual licensed under the MIT and the University of Illinois Open +// Source Licenses. See LICENSE.txt for details. +// +//===----------------------------------------------------------------------===// +// +// Implementation of OpenMP API interface functions. +// +//===----------------------------------------------------------------------===// + +#include + +#include "device.h" +#include "private.h" +#include "rtl.h" + +#include +#include + +EXTERN int omp_get_num_devices(void) { + RTLsMtx.lock(); + size_t Devices_size = Devices.size(); + RTLsMtx.unlock(); + + DP("Call to omp_get_num_devices returning %zd\n", Devices_size); + + return Devices_size; +} + +EXTERN int omp_get_initial_device(void) { + DP("Call to omp_get_initial_device returning %d\n", HOST_DEVICE); + return HOST_DEVICE; +} + +EXTERN void *omp_target_alloc(size_t size, int device_num) { + DP("Call to omp_target_alloc for device %d requesting %zu bytes\n", + device_num, size); + + if (size <= 0) { + DP("Call to omp_target_alloc with non-positive length\n"); + return NULL; + } + + void *rc = NULL; + + if (device_num == omp_get_initial_device()) { + rc = malloc(size); + DP("omp_target_alloc returns host ptr " DPxMOD "\n", DPxPTR(rc)); + return rc; + } + + if (!device_is_ready(device_num)) { + DP("omp_target_alloc returns NULL ptr\n"); + return NULL; + } + + DeviceTy &Device = Devices[device_num]; + rc = Device.RTL->data_alloc(Device.RTLDeviceID, size, NULL); + DP("omp_target_alloc returns device ptr " DPxMOD "\n", DPxPTR(rc)); + return rc; +} + +EXTERN void omp_target_free(void *device_ptr, int device_num) { + DP("Call to omp_target_free for device %d and address " DPxMOD "\n", + device_num, DPxPTR(device_ptr)); + + if (!device_ptr) { + DP("Call to omp_target_free with NULL ptr\n"); + return; + } + + if (device_num == omp_get_initial_device()) { + free(device_ptr); + DP("omp_target_free deallocated host ptr\n"); + return; + } + + if (!device_is_ready(device_num)) { + DP("omp_target_free returns, nothing to do\n"); + return; + } + + DeviceTy &Device = Devices[device_num]; + Device.RTL->data_delete(Device.RTLDeviceID, (void *)device_ptr); + DP("omp_target_free deallocated device ptr\n"); +} + +EXTERN int omp_target_is_present(void *ptr, int device_num) { + DP("Call to omp_target_is_present for device %d and address " DPxMOD "\n", + device_num, DPxPTR(ptr)); + + if (!ptr) { + DP("Call to omp_target_is_present with NULL ptr, returning false\n"); + return false; + } + + if (device_num == omp_get_initial_device()) { + DP("Call to omp_target_is_present on host, returning true\n"); + return true; + } + + RTLsMtx.lock(); + size_t Devices_size = Devices.size(); + RTLsMtx.unlock(); + if (Devices_size <= (size_t)device_num) { + DP("Call to omp_target_is_present with invalid device ID, returning " + "false\n"); + return false; + } + + DeviceTy& Device = Devices[device_num]; + bool IsLast; // not used + int rc = (Device.getTgtPtrBegin(ptr, 0, IsLast, false) != NULL); + DP("Call to omp_target_is_present returns %d\n", rc); + return rc; +} + +EXTERN int omp_target_memcpy(void *dst, void *src, size_t length, + size_t dst_offset, size_t src_offset, int dst_device, int src_device) { + DP("Call to omp_target_memcpy, dst device %d, src device %d, " + "dst addr " DPxMOD ", src addr " DPxMOD ", dst offset %zu, " + "src offset %zu, length %zu\n", dst_device, src_device, DPxPTR(dst), + DPxPTR(src), dst_offset, src_offset, length); + + if (!dst || !src || length <= 0) { + DP("Call to omp_target_memcpy with invalid arguments\n"); + return OFFLOAD_FAIL; + } + + if (src_device != omp_get_initial_device() && !device_is_ready(src_device)) { + DP("omp_target_memcpy returns OFFLOAD_FAIL\n"); + return OFFLOAD_FAIL; + } + + if (dst_device != omp_get_initial_device() && !device_is_ready(dst_device)) { + DP("omp_target_memcpy returns OFFLOAD_FAIL\n"); + return OFFLOAD_FAIL; + } + + int rc = OFFLOAD_SUCCESS; + void *srcAddr = (char *)src + src_offset; + void *dstAddr = (char *)dst + dst_offset; + + if (src_device == omp_get_initial_device() && + dst_device == omp_get_initial_device()) { + DP("copy from host to host\n"); + const void *p = memcpy(dstAddr, srcAddr, length); + if (p == NULL) + rc = OFFLOAD_FAIL; + } else if (src_device == omp_get_initial_device()) { + DP("copy from host to device\n"); + DeviceTy& DstDev = Devices[dst_device]; + rc = DstDev.data_submit(dstAddr, srcAddr, length); + } else if (dst_device == omp_get_initial_device()) { + DP("copy from device to host\n"); + DeviceTy& SrcDev = Devices[src_device]; + rc = SrcDev.data_retrieve(dstAddr, srcAddr, length); + } else { + DP("copy from device to device\n"); + void *buffer = malloc(length); + DeviceTy& SrcDev = Devices[src_device]; + DeviceTy& DstDev = Devices[dst_device]; + rc = SrcDev.data_retrieve(buffer, srcAddr, length); + if (rc == OFFLOAD_SUCCESS) + rc = DstDev.data_submit(dstAddr, buffer, length); + } + + DP("omp_target_memcpy returns %d\n", rc); + return rc; +} + +EXTERN int omp_target_memcpy_rect(void *dst, void *src, size_t element_size, + int num_dims, const size_t *volume, const size_t *dst_offsets, + const size_t *src_offsets, const size_t *dst_dimensions, + const size_t *src_dimensions, int dst_device, int src_device) { + DP("Call to omp_target_memcpy_rect, dst device %d, src device %d, " + "dst addr " DPxMOD ", src addr " DPxMOD ", dst offsets " DPxMOD ", " + "src offsets " DPxMOD ", dst dims " DPxMOD ", src dims " DPxMOD ", " + "volume " DPxMOD ", element size %zu, num_dims %d\n", dst_device, + src_device, DPxPTR(dst), DPxPTR(src), DPxPTR(dst_offsets), + DPxPTR(src_offsets), DPxPTR(dst_dimensions), DPxPTR(src_dimensions), + DPxPTR(volume), element_size, num_dims); + + if (!(dst || src)) { + DP("Call to omp_target_memcpy_rect returns max supported dimensions %d\n", + INT_MAX); + return INT_MAX; + } + + if (!dst || !src || element_size < 1 || num_dims < 1 || !volume || + !dst_offsets || !src_offsets || !dst_dimensions || !src_dimensions) { + DP("Call to omp_target_memcpy_rect with invalid arguments\n"); + return OFFLOAD_FAIL; + } + + int rc; + if (num_dims == 1) { + rc = omp_target_memcpy(dst, src, element_size * volume[0], + element_size * dst_offsets[0], element_size * src_offsets[0], + dst_device, src_device); + } else { + size_t dst_slice_size = element_size; + size_t src_slice_size = element_size; + for (int i=1; i DevicesTy; extern DevicesTy Devices; +extern bool device_is_ready(int device_num); + #endif Index: openmp/trunk/libomptarget/src/device.cpp =================================================================== --- openmp/trunk/libomptarget/src/device.cpp +++ openmp/trunk/libomptarget/src/device.cpp @@ -332,3 +332,34 @@ return RTL->run_team_region(RTLDeviceID, TgtEntryPtr, TgtVarsPtr, TgtOffsets, TgtVarsSize, NumTeams, ThreadLimit, LoopTripCount); } + +/// Check whether a device has an associated RTL and initialize it if it's not +/// already initialized. +bool device_is_ready(int device_num) { + DP("Checking whether device %d is ready.\n", device_num); + // Devices.size() can only change while registering a new + // library, so try to acquire the lock of RTLs' mutex. + RTLsMtx.lock(); + size_t Devices_size = Devices.size(); + RTLsMtx.unlock(); + if (Devices_size <= (size_t)device_num) { + DP("Device ID %d does not have a matching RTL\n", device_num); + return false; + } + + // Get device info + DeviceTy &Device = Devices[device_num]; + + DP("Is the device %d (local ID %d) initialized? %d\n", device_num, + Device.RTLDeviceID, Device.IsInit); + + // Init the device if not done before + if (!Device.IsInit && Device.initOnce() != OFFLOAD_SUCCESS) { + DP("Failed to init device %d\n", device_num); + return false; + } + + DP("Device %d is ready to use.\n", device_num); + + return true; +} Index: openmp/trunk/libomptarget/src/omptarget.cpp =================================================================== --- openmp/trunk/libomptarget/src/omptarget.cpp +++ openmp/trunk/libomptarget/src/omptarget.cpp @@ -19,309 +19,12 @@ #include "rtl.h" #include -#include -#include #include #ifdef OMPTARGET_DEBUG int DebugLevel = 0; #endif // OMPTARGET_DEBUG -/// Check whether a device has an associated RTL and initialize it if it's not -/// already initialized. -static bool device_is_ready(int device_num) { - DP("Checking whether device %d is ready.\n", device_num); - // Devices.size() can only change while registering a new - // library, so try to acquire the lock of RTLs' mutex. - RTLsMtx.lock(); - size_t Devices_size = Devices.size(); - RTLsMtx.unlock(); - if (Devices_size <= (size_t)device_num) { - DP("Device ID %d does not have a matching RTL\n", device_num); - return false; - } - - // Get device info - DeviceTy &Device = Devices[device_num]; - - DP("Is the device %d (local ID %d) initialized? %d\n", device_num, - Device.RTLDeviceID, Device.IsInit); - - // Init the device if not done before - if (!Device.IsInit && Device.initOnce() != OFFLOAD_SUCCESS) { - DP("Failed to init device %d\n", device_num); - return false; - } - - DP("Device %d is ready to use.\n", device_num); - - return true; -} - -//////////////////////////////////////////////////////////////////////////////// -// Target API functions -// -EXTERN int omp_get_num_devices(void) { - RTLsMtx.lock(); - size_t Devices_size = Devices.size(); - RTLsMtx.unlock(); - - DP("Call to omp_get_num_devices returning %zd\n", Devices_size); - - return Devices_size; -} - -EXTERN int omp_get_initial_device(void) { - DP("Call to omp_get_initial_device returning %d\n", HOST_DEVICE); - return HOST_DEVICE; -} - -EXTERN void *omp_target_alloc(size_t size, int device_num) { - DP("Call to omp_target_alloc for device %d requesting %zu bytes\n", - device_num, size); - - if (size <= 0) { - DP("Call to omp_target_alloc with non-positive length\n"); - return NULL; - } - - void *rc = NULL; - - if (device_num == omp_get_initial_device()) { - rc = malloc(size); - DP("omp_target_alloc returns host ptr " DPxMOD "\n", DPxPTR(rc)); - return rc; - } - - if (!device_is_ready(device_num)) { - DP("omp_target_alloc returns NULL ptr\n"); - return NULL; - } - - DeviceTy &Device = Devices[device_num]; - rc = Device.RTL->data_alloc(Device.RTLDeviceID, size, NULL); - DP("omp_target_alloc returns device ptr " DPxMOD "\n", DPxPTR(rc)); - return rc; -} - -EXTERN void omp_target_free(void *device_ptr, int device_num) { - DP("Call to omp_target_free for device %d and address " DPxMOD "\n", - device_num, DPxPTR(device_ptr)); - - if (!device_ptr) { - DP("Call to omp_target_free with NULL ptr\n"); - return; - } - - if (device_num == omp_get_initial_device()) { - free(device_ptr); - DP("omp_target_free deallocated host ptr\n"); - return; - } - - if (!device_is_ready(device_num)) { - DP("omp_target_free returns, nothing to do\n"); - return; - } - - DeviceTy &Device = Devices[device_num]; - Device.RTL->data_delete(Device.RTLDeviceID, (void *)device_ptr); - DP("omp_target_free deallocated device ptr\n"); -} - -EXTERN int omp_target_is_present(void *ptr, int device_num) { - DP("Call to omp_target_is_present for device %d and address " DPxMOD "\n", - device_num, DPxPTR(ptr)); - - if (!ptr) { - DP("Call to omp_target_is_present with NULL ptr, returning false\n"); - return false; - } - - if (device_num == omp_get_initial_device()) { - DP("Call to omp_target_is_present on host, returning true\n"); - return true; - } - - RTLsMtx.lock(); - size_t Devices_size = Devices.size(); - RTLsMtx.unlock(); - if (Devices_size <= (size_t)device_num) { - DP("Call to omp_target_is_present with invalid device ID, returning " - "false\n"); - return false; - } - - DeviceTy& Device = Devices[device_num]; - bool IsLast; // not used - int rc = (Device.getTgtPtrBegin(ptr, 0, IsLast, false) != NULL); - DP("Call to omp_target_is_present returns %d\n", rc); - return rc; -} - -EXTERN int omp_target_memcpy(void *dst, void *src, size_t length, - size_t dst_offset, size_t src_offset, int dst_device, int src_device) { - DP("Call to omp_target_memcpy, dst device %d, src device %d, " - "dst addr " DPxMOD ", src addr " DPxMOD ", dst offset %zu, " - "src offset %zu, length %zu\n", dst_device, src_device, DPxPTR(dst), - DPxPTR(src), dst_offset, src_offset, length); - - if (!dst || !src || length <= 0) { - DP("Call to omp_target_memcpy with invalid arguments\n"); - return OFFLOAD_FAIL; - } - - if (src_device != omp_get_initial_device() && !device_is_ready(src_device)) { - DP("omp_target_memcpy returns OFFLOAD_FAIL\n"); - return OFFLOAD_FAIL; - } - - if (dst_device != omp_get_initial_device() && !device_is_ready(dst_device)) { - DP("omp_target_memcpy returns OFFLOAD_FAIL\n"); - return OFFLOAD_FAIL; - } - - int rc = OFFLOAD_SUCCESS; - void *srcAddr = (char *)src + src_offset; - void *dstAddr = (char *)dst + dst_offset; - - if (src_device == omp_get_initial_device() && - dst_device == omp_get_initial_device()) { - DP("copy from host to host\n"); - const void *p = memcpy(dstAddr, srcAddr, length); - if (p == NULL) - rc = OFFLOAD_FAIL; - } else if (src_device == omp_get_initial_device()) { - DP("copy from host to device\n"); - DeviceTy& DstDev = Devices[dst_device]; - rc = DstDev.data_submit(dstAddr, srcAddr, length); - } else if (dst_device == omp_get_initial_device()) { - DP("copy from device to host\n"); - DeviceTy& SrcDev = Devices[src_device]; - rc = SrcDev.data_retrieve(dstAddr, srcAddr, length); - } else { - DP("copy from device to device\n"); - void *buffer = malloc(length); - DeviceTy& SrcDev = Devices[src_device]; - DeviceTy& DstDev = Devices[dst_device]; - rc = SrcDev.data_retrieve(buffer, srcAddr, length); - if (rc == OFFLOAD_SUCCESS) - rc = DstDev.data_submit(dstAddr, buffer, length); - } - - DP("omp_target_memcpy returns %d\n", rc); - return rc; -} - -EXTERN int omp_target_memcpy_rect(void *dst, void *src, size_t element_size, - int num_dims, const size_t *volume, const size_t *dst_offsets, - const size_t *src_offsets, const size_t *dst_dimensions, - const size_t *src_dimensions, int dst_device, int src_device) { - DP("Call to omp_target_memcpy_rect, dst device %d, src device %d, " - "dst addr " DPxMOD ", src addr " DPxMOD ", dst offsets " DPxMOD ", " - "src offsets " DPxMOD ", dst dims " DPxMOD ", src dims " DPxMOD ", " - "volume " DPxMOD ", element size %zu, num_dims %d\n", dst_device, - src_device, DPxPTR(dst), DPxPTR(src), DPxPTR(dst_offsets), - DPxPTR(src_offsets), DPxPTR(dst_dimensions), DPxPTR(src_dimensions), - DPxPTR(volume), element_size, num_dims); - - if (!(dst || src)) { - DP("Call to omp_target_memcpy_rect returns max supported dimensions %d\n", - INT_MAX); - return INT_MAX; - } - - if (!dst || !src || element_size < 1 || num_dims < 1 || !volume || - !dst_offsets || !src_offsets || !dst_dimensions || !src_dimensions) { - DP("Call to omp_target_memcpy_rect with invalid arguments\n"); - return OFFLOAD_FAIL; - } - - int rc; - if (num_dims == 1) { - rc = omp_target_memcpy(dst, src, element_size * volume[0], - element_size * dst_offsets[0], element_size * src_offsets[0], - dst_device, src_device); - } else { - size_t dst_slice_size = element_size; - size_t src_slice_size = element_size; - for (int i=1; i