diff --git a/llvm/include/llvm/ExecutionEngine/Orc/MemoryMapper.h b/llvm/include/llvm/ExecutionEngine/Orc/MemoryMapper.h --- a/llvm/include/llvm/ExecutionEngine/Orc/MemoryMapper.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/MemoryMapper.h @@ -109,6 +109,47 @@ AllocationMap Allocations; }; +class SharedMemoryMapper final : public MemoryMapper { +public: + struct SymbolAddrs { + ExecutorAddr Instance; + ExecutorAddr Reserve; + ExecutorAddr Initialize; + ExecutorAddr Deinitialize; + ExecutorAddr Release; + }; + + SharedMemoryMapper(ExecutorProcessControl &EPC, SymbolAddrs SAs) + : EPC(EPC), SAs(SAs) {} + + void reserve(size_t NumBytes, OnReservedFunction OnReserved) override; + + char *prepare(ExecutorAddr Addr, size_t ContentSize) override; + + void initialize(AllocInfo &AI, OnInitializedFunction OnInitialized) override; + + void deinitialize(ArrayRef Allocations, + OnDeinitializedFunction OnDeInitialized) override; + + void release(ArrayRef Reservations, + OnReleasedFunction OnRelease) override; + + ~SharedMemoryMapper() override; + +private: + struct Reservation { + void *LocalAddr; + size_t Size; + }; + + ExecutorProcessControl &EPC; + SymbolAddrs SAs; + + std::mutex Mutex; + + std::map Reservations; +}; + } // namespace orc } // end namespace llvm diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h --- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h @@ -31,6 +31,12 @@ extern const char *SimpleExecutorMemoryManagerFinalizeWrapperName; extern const char *SimpleExecutorMemoryManagerDeallocateWrapperName; +extern const char *ExecutorSharedMemoryMapperServiceInstanceName; +extern const char *ExecutorSharedMemoryMapperServiceReserveWrapperName; +extern const char *ExecutorSharedMemoryMapperServiceInitializeWrapperName; +extern const char *ExecutorSharedMemoryMapperServiceDeinitializeWrapperName; +extern const char *ExecutorSharedMemoryMapperServiceReleaseWrapperName; + extern const char *MemoryWriteUInt8sWrapperName; extern const char *MemoryWriteUInt16sWrapperName; extern const char *MemoryWriteUInt32sWrapperName; @@ -58,6 +64,21 @@ using SPSSimpleExecutorMemoryManagerDeallocateSignature = shared::SPSError( shared::SPSExecutorAddr, shared::SPSSequence); +// ExecutorSharedMemoryMapperService +using SPSExecutorSharedMemoryMapperServiceReserveSignature = + shared::SPSExpected< + shared::SPSTuple>( + shared::SPSExecutorAddr, uint64_t); +using SPSExecutorSharedMemoryMapperServiceInitializeSignature = + shared::SPSExpected( + shared::SPSExecutorAddr, shared::SPSExecutorAddr, + shared::SPSSharedMemoryFinalizeRequest); +using SPSExecutorSharedMemoryMapperServiceDeinitializeSignature = + shared::SPSError(shared::SPSExecutorAddr, + shared::SPSSequence); +using SPSExecutorSharedMemoryMapperServiceReleaseSignature = shared::SPSError( + shared::SPSExecutorAddr, shared::SPSSequence); + using SPSRunAsMainSignature = int64_t(shared::SPSExecutorAddr, shared::SPSSequence); diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h --- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h @@ -82,6 +82,17 @@ shared::AllocActions Actions; }; +struct SharedMemorySegFinalizeRequest { + WireProtectionFlags Prot; + ExecutorAddr Addr; + uint64_t Size; +}; + +struct SharedMemoryFinalizeRequest { + std::vector Segments; + shared::AllocActions Actions; +}; + template struct UIntWrite { UIntWrite() = default; UIntWrite(ExecutorAddr Addr, T Value) : Addr(Addr), Value(Value) {} @@ -131,6 +142,13 @@ using SPSFinalizeRequest = SPSTuple, SPSSequence>; +using SPSSharedMemorySegFinalizeRequest = + SPSTuple; + +using SPSSharedMemoryFinalizeRequest = + SPSTuple, + SPSSequence>; + template using SPSMemoryAccessUIntWrite = SPSTuple; @@ -204,6 +222,48 @@ } }; +template <> +class SPSSerializationTraits { + using SFRAL = SPSSharedMemorySegFinalizeRequest::AsArgList; + +public: + static size_t size(const tpctypes::SharedMemorySegFinalizeRequest &SFR) { + return SFRAL::size(SFR.Prot, SFR.Addr, SFR.Size); + } + + static bool serialize(SPSOutputBuffer &OB, + const tpctypes::SharedMemorySegFinalizeRequest &SFR) { + return SFRAL::serialize(OB, SFR.Prot, SFR.Addr, SFR.Size); + } + + static bool deserialize(SPSInputBuffer &IB, + tpctypes::SharedMemorySegFinalizeRequest &SFR) { + return SFRAL::deserialize(IB, SFR.Prot, SFR.Addr, SFR.Size); + } +}; + +template <> +class SPSSerializationTraits { + using FRAL = SPSSharedMemoryFinalizeRequest::AsArgList; + +public: + static size_t size(const tpctypes::SharedMemoryFinalizeRequest &FR) { + return FRAL::size(FR.Segments, FR.Actions); + } + + static bool serialize(SPSOutputBuffer &OB, + const tpctypes::SharedMemoryFinalizeRequest &FR) { + return FRAL::serialize(OB, FR.Segments, FR.Actions); + } + + static bool deserialize(SPSInputBuffer &IB, + tpctypes::SharedMemoryFinalizeRequest &FR) { + return FRAL::deserialize(IB, FR.Segments, FR.Actions); + } +}; + template class SPSSerializationTraits, tpctypes::UIntWrite> { @@ -244,7 +304,6 @@ } }; - } // end namespace shared } // end namespace orc } // end namespace llvm diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h @@ -0,0 +1,78 @@ +//===----------- ExecutorSharedMemoryMapperService.h ------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_EXECUTORSHAREDMEMORYMAPPERSERVICE +#define LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_EXECUTORSHAREDMEMORYMAPPERSERVICE + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h" +#include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorBootstrapService.h" + +#include +#include + +#if defined(_WIN32) +#include +#endif + +namespace llvm { +namespace orc { +namespace rt_bootstrap { + +class ExecutorSharedMemoryMapperService final + : public ExecutorBootstrapService { +public: + ~ExecutorSharedMemoryMapperService(){}; + + Expected> reserve(uint64_t Size); + Expected initialize(ExecutorAddr Reservation, + tpctypes::SharedMemoryFinalizeRequest &FR); + + Error deinitialize(const std::vector &Bases); + Error release(const std::vector &Bases); + + Error shutdown() override; + void addBootstrapSymbols(StringMap &M) override; + +private: + struct Allocation { + std::vector DeinitializationActions; + }; + using AllocationMap = DenseMap; + + struct Reservation { + size_t Size; + std::vector Allocations; +#if defined(_WIN32) + HANDLE SharedMemoryFile; +#endif + }; + using ReservationMap = DenseMap; + + static llvm::orc::shared::CWrapperFunctionResult + reserveWrapper(const char *ArgData, size_t ArgSize); + + static llvm::orc::shared::CWrapperFunctionResult + initializeWrapper(const char *ArgData, size_t ArgSize); + + static llvm::orc::shared::CWrapperFunctionResult + deinitializeWrapper(const char *ArgData, size_t ArgSize); + + static llvm::orc::shared::CWrapperFunctionResult + releaseWrapper(const char *ArgData, size_t ArgSize); + + std::atomic SharedMemoryCount{0}; + std::mutex Mutex; + ReservationMap Reservations; + AllocationMap Allocations; +}; + +} // namespace rt_bootstrap +} // namespace orc +} // namespace llvm +#endif // LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_EXECUTORSHAREDMEMORYMAPPERSERVICE diff --git a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt --- a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt +++ b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt @@ -1,3 +1,7 @@ +if( CMAKE_HOST_UNIX AND HAVE_LIBRT ) + set(rt_lib rt) +endif() + add_llvm_component_library(LLVMOrcJIT CompileOnDemandLayer.cpp CompileUtils.cpp @@ -45,6 +49,7 @@ LINK_LIBS ${LLVM_PTHREAD_LIB} + ${rt_lib} LINK_COMPONENTS Core diff --git a/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp b/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp --- a/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp +++ b/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp @@ -8,6 +8,17 @@ #include "llvm/ExecutionEngine/Orc/MemoryMapper.h" +#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" +#include "llvm/Support/WindowsError.h" + +#if defined(LLVM_ON_UNIX) +#include +#include +#include +#elif defined(_WIN32) +#include +#endif + namespace llvm { namespace orc { @@ -147,6 +158,225 @@ cantFail(F.get()); } +// SharedMemoryMapper + +void SharedMemoryMapper::reserve(size_t NumBytes, + OnReservedFunction OnReserved) { +#if defined(LLVM_ON_UNIX) || defined(_WIN32) + + EPC.callSPSWrapperAsync< + rt::SPSExecutorSharedMemoryMapperServiceReserveSignature>( + SAs.Reserve, + [this, NumBytes, OnReserved = std::move(OnReserved)]( + Error SerializationErr, + Expected> Result) mutable { + if (SerializationErr) { + cantFail(Result.takeError()); + return OnReserved(std::move(SerializationErr)); + } + + if (!Result) + return OnReserved(Result.takeError()); + + ExecutorAddr RemoteAddr; + std::string SharedMemoryName; + std::tie(RemoteAddr, SharedMemoryName) = std::move(*Result); + + void *LocalAddr = nullptr; + +#if defined(LLVM_ON_UNIX) + + int SharedMemoryFile = shm_open(SharedMemoryName.c_str(), O_RDWR, 0700); + if (SharedMemoryFile < 0) { + return OnReserved(errorCodeToError( + std::error_code(errno, std::generic_category()))); + } + + // this prevents other processes from accessing it by name + shm_unlink(SharedMemoryName.c_str()); + + LocalAddr = mmap(nullptr, NumBytes, PROT_READ | PROT_WRITE, MAP_SHARED, + SharedMemoryFile, 0); + if (LocalAddr == MAP_FAILED) { + return OnReserved(errorCodeToError( + std::error_code(errno, std::generic_category()))); + } + + close(SharedMemoryFile); + +#elif defined(_WIN32) + + std::wstring WideSharedMemoryName(SharedMemoryName.begin(), + SharedMemoryName.end()); + HANDLE SharedMemoryFile = OpenFileMappingW(FILE_MAP_ALL_ACCESS, FALSE, + WideSharedMemoryName.c_str()); + if (!SharedMemoryFile) + return OnReserved(errorCodeToError(mapWindowsError(GetLastError()))); + + LocalAddr = + MapViewOfFile(SharedMemoryFile, FILE_MAP_ALL_ACCESS, 0, 0, 0); + if (!LocalAddr) { + CloseHandle(SharedMemoryFile); + return OnReserved(errorCodeToError(mapWindowsError(GetLastError()))); + } + + CloseHandle(SharedMemoryFile); + +#endif + { + std::lock_guard Lock(Mutex); + Reservations.insert({RemoteAddr, {LocalAddr, NumBytes}}); + } + + OnReserved(ExecutorAddrRange(RemoteAddr, NumBytes)); + }, + SAs.Instance, static_cast(NumBytes)); + +#else + OnReserved(make_error( + "SharedMemoryMapper is not supported on this platform yet", + inconvertibleErrorCode())); +#endif +} + +char *SharedMemoryMapper::prepare(ExecutorAddr Addr, size_t ContentSize) { + auto R = Reservations.upper_bound(Addr); + assert(R != Reservations.begin() && "Attempt to prepare unknown range"); + R--; + + ExecutorAddrDiff Offset = Addr - R->first; + + return static_cast(R->second.LocalAddr) + Offset; +} + +void SharedMemoryMapper::initialize(MemoryMapper::AllocInfo &AI, + OnInitializedFunction OnInitialized) { + auto Reservation = Reservations.find(AI.MappingBase); + assert(Reservation != Reservations.end() && + "Attempt to initialize unreserved range"); + + tpctypes::SharedMemoryFinalizeRequest FR; + + AI.Actions.swap(FR.Actions); + + FR.Segments.reserve(AI.Segments.size()); + + for (auto Segment : AI.Segments) { + char *Base = + static_cast(Reservation->second.LocalAddr) + Segment.Offset; + std::memset(Base + Segment.ContentSize, 0, Segment.ZeroFillSize); + + tpctypes::SharedMemorySegFinalizeRequest SegReq; + SegReq.Prot = tpctypes::toWireProtectionFlags( + static_cast(Segment.Prot)); + SegReq.Addr = AI.MappingBase + Segment.Offset; + SegReq.Size = Segment.ContentSize + Segment.ZeroFillSize; + + FR.Segments.push_back(SegReq); + } + + EPC.callSPSWrapperAsync< + rt::SPSExecutorSharedMemoryMapperServiceInitializeSignature>( + SAs.Initialize, + [OnInitialized = std::move(OnInitialized)]( + Error SerializationErr, Expected Result) mutable { + if (SerializationErr) { + cantFail(Result.takeError()); + return OnInitialized(std::move(SerializationErr)); + } + + OnInitialized(std::move(Result)); + }, + SAs.Instance, AI.MappingBase, std::move(FR)); +} + +void SharedMemoryMapper::deinitialize( + ArrayRef Allocations, + MemoryMapper::OnDeinitializedFunction OnDeinitialized) { + EPC.callSPSWrapperAsync< + rt::SPSExecutorSharedMemoryMapperServiceDeinitializeSignature>( + SAs.Deinitialize, + [OnDeinitialized = std::move(OnDeinitialized)](Error SerializationErr, + Error Result) mutable { + if (SerializationErr) { + cantFail(std::move(Result)); + return OnDeinitialized(std::move(SerializationErr)); + } + + OnDeinitialized(std::move(Result)); + }, + SAs.Instance, Allocations); +} + +void SharedMemoryMapper::release(ArrayRef Bases, + OnReleasedFunction OnReleased) { +#if defined(LLVM_ON_UNIX) || defined(_WIN32) + Error Err = Error::success(); + + { + std::lock_guard Lock(Mutex); + + for (auto Base : Bases) { + +#if defined(LLVM_ON_UNIX) + + if (munmap(Reservations[Base].LocalAddr, Reservations[Base].Size) != 0) + Err = joinErrors(std::move(Err), errorCodeToError(std::error_code( + errno, std::generic_category()))); + +#elif defined(_WIN32) + + if (!UnmapViewOfFile(Reservations[Base].LocalAddr)) + joinErrors(std::move(Err), + errorCodeToError(mapWindowsError(GetLastError()))); + +#endif + + Reservations.erase(Base); + } + } + + EPC.callSPSWrapperAsync< + rt::SPSExecutorSharedMemoryMapperServiceReleaseSignature>( + SAs.Release, + [OnReleased = std::move(OnReleased), + Err = std::move(Err)](Error SerializationErr, Error Result) mutable { + if (SerializationErr) { + cantFail(std::move(Result)); + return OnReleased( + joinErrors(std::move(Err), std::move(SerializationErr))); + } + + return OnReleased(joinErrors(std::move(Err), std::move(Result))); + }, + SAs.Instance, Bases); +#else + OnReleased(make_error( + "SharedMemoryMapper is not supported on this platform yet", + inconvertibleErrorCode())); +#endif +} + +SharedMemoryMapper::~SharedMemoryMapper() { + std::vector ReservationAddrs; + if (!Reservations.empty()) { + std::lock_guard Lock(Mutex); + { + ReservationAddrs.reserve(Reservations.size()); + for (const auto &R : Reservations) { + ReservationAddrs.push_back(R.first); + } + } + } + + std::promise P; + auto F = P.get_future(); + release(ReservationAddrs, [&](Error Err) { P.set_value(std::move(Err)); }); + // FIXME: Release can actually fail. The error should be propagated. + // Meanwhile, a better option is to explicitly call release(). + cantFail(F.get()); +} + } // namespace orc } // namespace llvm diff --git a/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp b/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp --- a/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp +++ b/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp @@ -18,6 +18,7 @@ "__llvm_orc_SimpleExecutorDylibManager_open_wrapper"; const char *SimpleExecutorDylibManagerLookupWrapperName = "__llvm_orc_SimpleExecutorDylibManager_lookup_wrapper"; + const char *SimpleExecutorMemoryManagerInstanceName = "__llvm_orc_SimpleExecutorMemoryManager_Instance"; const char *SimpleExecutorMemoryManagerReserveWrapperName = @@ -26,6 +27,18 @@ "__llvm_orc_SimpleExecutorMemoryManager_finalize_wrapper"; const char *SimpleExecutorMemoryManagerDeallocateWrapperName = "__llvm_orc_SimpleExecutorMemoryManager_deallocate_wrapper"; + +const char *ExecutorSharedMemoryMapperServiceInstanceName = + "__llvm_orc_ExecutorSharedMemoryMapperService_Instance"; +const char *ExecutorSharedMemoryMapperServiceReserveWrapperName = + "__llvm_orc_ExecutorSharedMemoryMapperService_Reserve"; +const char *ExecutorSharedMemoryMapperServiceInitializeWrapperName = + "__llvm_orc_ExecutorSharedMemoryMapperService_Initialize"; +const char *ExecutorSharedMemoryMapperServiceDeinitializeWrapperName = + "__llvm_orc_ExecutorSharedMemoryMapperService_Deinitialize"; +const char *ExecutorSharedMemoryMapperServiceReleaseWrapperName = + "__llvm_orc_ExecutorSharedMemoryMapperService_Release"; + const char *MemoryWriteUInt8sWrapperName = "__llvm_orc_bootstrap_mem_write_uint8s_wrapper"; const char *MemoryWriteUInt16sWrapperName = @@ -36,10 +49,12 @@ "__llvm_orc_bootstrap_mem_write_uint64s_wrapper"; const char *MemoryWriteBuffersWrapperName = "__llvm_orc_bootstrap_mem_write_buffers_wrapper"; + const char *RegisterEHFrameSectionWrapperName = "__llvm_orc_bootstrap_register_ehframe_section_wrapper"; const char *DeregisterEHFrameSectionWrapperName = "__llvm_orc_bootstrap_deregister_ehframe_section_wrapper"; + const char *RunAsMainWrapperName = "__llvm_orc_bootstrap_run_as_main_wrapper"; } // end namespace rt diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt --- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt +++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt @@ -1,4 +1,9 @@ +if( CMAKE_HOST_UNIX AND HAVE_LIBRT ) + set(rt_lib rt) +endif() + add_llvm_component_library(LLVMOrcTargetProcess + ExecutorSharedMemoryMapperService.cpp JITLoaderGDB.cpp OrcRTBootstrap.cpp RegisterEHFrames.cpp @@ -12,6 +17,7 @@ LINK_LIBS ${LLVM_PTHREAD_LIB} + ${rt_lib} LINK_COMPONENTS OrcShared diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp @@ -0,0 +1,341 @@ +//===---------- ExecutorSharedMemoryMapperService.cpp -----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h" + +#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" +#include "llvm/Support/Process.h" +#include "llvm/Support/WindowsError.h" + +#include + +#if defined(LLVM_ON_UNIX) +#include +#include +#include +#include +#endif + +#if defined(_WIN32) +static DWORD getWindowsProtectionFlags(unsigned Flags) { + switch (Flags & llvm::sys::Memory::MF_RWE_MASK) { + case llvm::sys::Memory::MF_READ: + return PAGE_READONLY; + case llvm::sys::Memory::MF_WRITE: + // Note: PAGE_WRITE is not supported by VirtualProtect + return PAGE_READWRITE; + case llvm::sys::Memory::MF_READ | llvm::sys::Memory::MF_WRITE: + return PAGE_READWRITE; + case llvm::sys::Memory::MF_READ | llvm::sys::Memory::MF_EXEC: + return PAGE_EXECUTE_READ; + case llvm::sys::Memory::MF_READ | llvm::sys::Memory::MF_WRITE | + llvm::sys::Memory::MF_EXEC: + return PAGE_EXECUTE_READWRITE; + case llvm::sys::Memory::MF_EXEC: + return PAGE_EXECUTE; + default: + llvm_unreachable("Illegal memory protection flag specified!"); + } + // Provide a default return value as required by some compilers. + return PAGE_NOACCESS; +} +#endif + +namespace llvm { +namespace orc { +namespace rt_bootstrap { + +Expected> +ExecutorSharedMemoryMapperService::reserve(uint64_t Size) { +#if defined(LLVM_ON_UNIX) || defined(_WIN32) + +#if defined(LLVM_ON_UNIX) + + std::string SharedMemoryName; + { + std::stringstream SharedMemoryNameStream; + SharedMemoryNameStream << "/jitlink_" << sys::Process::getProcessId() << '_' + << (++SharedMemoryCount); + SharedMemoryName = SharedMemoryNameStream.str(); + } + + int SharedMemoryFile = + shm_open(SharedMemoryName.c_str(), O_RDWR | O_CREAT | O_EXCL, 0700); + if (SharedMemoryFile < 0) + return errorCodeToError(std::error_code(errno, std::generic_category())); + + // by default size is 0 + if (ftruncate(SharedMemoryFile, Size) < 0) + return errorCodeToError(std::error_code(errno, std::generic_category())); + + void *Addr = mmap(nullptr, Size, PROT_NONE, MAP_SHARED, SharedMemoryFile, 0); + if (Addr == MAP_FAILED) + return errorCodeToError(std::error_code(errno, std::generic_category())); + + close(SharedMemoryFile); + +#elif defined(_WIN32) + + std::string SharedMemoryName; + { + std::stringstream SharedMemoryNameStream; + SharedMemoryNameStream << "jitlink_" << sys::Process::getProcessId() << '_' + << (++SharedMemoryCount); + SharedMemoryName = SharedMemoryNameStream.str(); + } + + std::wstring WideSharedMemoryName(SharedMemoryName.begin(), + SharedMemoryName.end()); + HANDLE SharedMemoryFile = CreateFileMappingW( + INVALID_HANDLE_VALUE, NULL, PAGE_EXECUTE_READWRITE, Size >> 32, + Size & 0xffffffff, WideSharedMemoryName.c_str()); + if (!SharedMemoryFile) + return errorCodeToError(mapWindowsError(GetLastError())); + + void *Addr = MapViewOfFile(SharedMemoryFile, + FILE_MAP_ALL_ACCESS | FILE_MAP_EXECUTE, 0, 0, 0); + if (!Addr) { + CloseHandle(SharedMemoryFile); + return errorCodeToError(mapWindowsError(GetLastError())); + } + +#endif + + { + std::lock_guard Lock(Mutex); + Reservations[Addr].Size = Size; +#if defined(_WIN32) + Reservations[Addr].SharedMemoryFile = SharedMemoryFile; +#endif + } + + return std::make_pair(ExecutorAddr::fromPtr(Addr), + std::move(SharedMemoryName)); +#else + return make_error( + "SharedMemoryMapper is not supported on this platform yet", + inconvertibleErrorCode()); +#endif +} + +Expected ExecutorSharedMemoryMapperService::initialize( + ExecutorAddr Reservation, tpctypes::SharedMemoryFinalizeRequest &FR) { +#if defined(LLVM_ON_UNIX) || defined(_WIN32) + + ExecutorAddr MinAddr(~0ULL); + + // Contents are already in place + for (auto &Segment : FR.Segments) { + if (Segment.Addr < MinAddr) + MinAddr = Segment.Addr; + +#if defined(LLVM_ON_UNIX) + + int NativeProt = 0; + if (Segment.Prot & tpctypes::WPF_Read) + NativeProt |= PROT_READ; + if (Segment.Prot & tpctypes::WPF_Write) + NativeProt |= PROT_WRITE; + if (Segment.Prot & tpctypes::WPF_Exec) + NativeProt |= PROT_EXEC; + + if (mprotect(Segment.Addr.toPtr(), Segment.Size, NativeProt)) + return errorCodeToError(std::error_code(errno, std::generic_category())); + +#elif defined(_WIN32) + + DWORD NativeProt = + getWindowsProtectionFlags(fromWireProtectionFlags(Segment.Prot)); + + if (!VirtualProtect(Segment.Addr.toPtr(), Segment.Size, NativeProt, + &NativeProt)) + return errorCodeToError(mapWindowsError(GetLastError())); + +#endif + + if (Segment.Prot & tpctypes::WPF_Exec) + sys::Memory::InvalidateInstructionCache(Segment.Addr.toPtr(), + Segment.Size); + } + + // Run finalization actions and get deinitlization action list. + auto DeinitializeActions = shared::runFinalizeActions(FR.Actions); + if (!DeinitializeActions) { + return DeinitializeActions.takeError(); + } + + { + std::lock_guard Lock(Mutex); + Allocations[MinAddr].DeinitializationActions = + std::move(*DeinitializeActions); + Reservations[Reservation.toPtr()].Allocations.push_back(MinAddr); + } + + return MinAddr; + +#else + return make_error( + "SharedMemoryMapper is not supported on this platform yet", + inconvertibleErrorCode()); +#endif +} + +Error ExecutorSharedMemoryMapperService::deinitialize( + const std::vector &Bases) { + Error AllErr = Error::success(); + + { + std::lock_guard Lock(Mutex); + + for (auto Base : Bases) { + if (Error Err = shared::runDeallocActions( + Allocations[Base].DeinitializationActions)) { + AllErr = joinErrors(std::move(AllErr), std::move(Err)); + } + + Allocations.erase(Base); + } + } + + return AllErr; +} + +Error ExecutorSharedMemoryMapperService::release( + const std::vector &Bases) { +#if defined(LLVM_ON_UNIX) || defined(_WIN32) + Error Err = Error::success(); + + for (auto Base : Bases) { + std::vector AllocAddrs; + size_t Size; + +#if defined(_WIN32) + HANDLE SharedMemoryFile; +#endif + + { + std::lock_guard Lock(Mutex); + auto &R = Reservations[Base.toPtr()]; + Size = R.Size; + +#if defined(_WIN32) + SharedMemoryFile = R.SharedMemoryFile; +#endif + + AllocAddrs.swap(R.Allocations); + } + + // deinitialize sub allocations + if (Error E = deinitialize(AllocAddrs)) + Err = joinErrors(std::move(Err), std::move(E)); + +#if defined(LLVM_ON_UNIX) + + if (munmap(Base.toPtr(), Size) != 0) + Err = joinErrors(std::move(Err), errorCodeToError(std::error_code( + errno, std::generic_category()))); + +#elif defined(_WIN32) + + if (!UnmapViewOfFile(Base.toPtr())) + Err = joinErrors(std::move(Err), + errorCodeToError(mapWindowsError(GetLastError()))); + + CloseHandle(SharedMemoryFile); + +#endif + + std::lock_guard Lock(Mutex); + Reservations.erase(Base.toPtr()); + } + + return Err; +#else + return make_error( + "SharedMemoryMapper is not supported on this platform yet", + inconvertibleErrorCode()); +#endif +} + +Error ExecutorSharedMemoryMapperService::shutdown() { + std::vector ReservationAddrs; + if (!Reservations.empty()) { + std::lock_guard Lock(Mutex); + { + ReservationAddrs.reserve(Reservations.size()); + for (const auto &R : Reservations) { + ReservationAddrs.push_back(ExecutorAddr::fromPtr(R.getFirst())); + } + } + } + return release(ReservationAddrs); + + return Error::success(); +} + +void ExecutorSharedMemoryMapperService::addBootstrapSymbols( + StringMap &M) { + M[rt::ExecutorSharedMemoryMapperServiceInstanceName] = + ExecutorAddr::fromPtr(this); + M[rt::ExecutorSharedMemoryMapperServiceReserveWrapperName] = + ExecutorAddr::fromPtr(&reserveWrapper); + M[rt::ExecutorSharedMemoryMapperServiceInitializeWrapperName] = + ExecutorAddr::fromPtr(&initializeWrapper); + M[rt::ExecutorSharedMemoryMapperServiceDeinitializeWrapperName] = + ExecutorAddr::fromPtr(&deinitializeWrapper); + M[rt::ExecutorSharedMemoryMapperServiceReleaseWrapperName] = + ExecutorAddr::fromPtr(&releaseWrapper); +} + +llvm::orc::shared::CWrapperFunctionResult +ExecutorSharedMemoryMapperService::reserveWrapper(const char *ArgData, + size_t ArgSize) { + return shared::WrapperFunction< + rt::SPSExecutorSharedMemoryMapperServiceReserveSignature>:: + handle(ArgData, ArgSize, + shared::makeMethodWrapperHandler( + &ExecutorSharedMemoryMapperService::reserve)) + .release(); +} + +llvm::orc::shared::CWrapperFunctionResult +ExecutorSharedMemoryMapperService::initializeWrapper(const char *ArgData, + size_t ArgSize) { + return shared::WrapperFunction< + rt::SPSExecutorSharedMemoryMapperServiceInitializeSignature>:: + handle(ArgData, ArgSize, + shared::makeMethodWrapperHandler( + &ExecutorSharedMemoryMapperService::initialize)) + .release(); +} + +llvm::orc::shared::CWrapperFunctionResult +ExecutorSharedMemoryMapperService::deinitializeWrapper(const char *ArgData, + size_t ArgSize) { + return shared::WrapperFunction< + rt::SPSExecutorSharedMemoryMapperServiceDeinitializeSignature>:: + handle(ArgData, ArgSize, + shared::makeMethodWrapperHandler( + &ExecutorSharedMemoryMapperService::deinitialize)) + .release(); +} + +llvm::orc::shared::CWrapperFunctionResult +ExecutorSharedMemoryMapperService::releaseWrapper(const char *ArgData, + size_t ArgSize) { + return shared::WrapperFunction< + rt::SPSExecutorSharedMemoryMapperServiceReleaseSignature>:: + handle(ArgData, ArgSize, + shared::makeMethodWrapperHandler( + &ExecutorSharedMemoryMapperService::release)) + .release(); +} + +} // namespace rt_bootstrap +} // end namespace orc +} // end namespace llvm diff --git a/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt b/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt --- a/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt +++ b/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt @@ -30,6 +30,7 @@ OrcTestCommon.cpp ResourceTrackerTest.cpp RTDyldObjectLinkingLayerTest.cpp + SharedMemoryMapperTest.cpp SimpleExecutorMemoryManagerTest.cpp SimplePackedSerializationTest.cpp SymbolStringPoolTest.cpp diff --git a/llvm/unittests/ExecutionEngine/Orc/SharedMemoryMapperTest.cpp b/llvm/unittests/ExecutionEngine/Orc/SharedMemoryMapperTest.cpp new file mode 100644 --- /dev/null +++ b/llvm/unittests/ExecutionEngine/Orc/SharedMemoryMapperTest.cpp @@ -0,0 +1,133 @@ +//===- SharedMemoryMapperTest.cpp -- Tests for SharedMemoryMapper ---------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "OrcTestCommon.h" + +#include "llvm/ExecutionEngine/Orc/MemoryMapper.h" +#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" +#include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h" +#include "llvm/Testing/Support/Error.h" + +using namespace llvm; +using namespace llvm::orc; +using namespace llvm::orc::shared; +using namespace llvm::orc::rt_bootstrap; + +#if defined(LLVM_ON_UNIX) || defined(_WIN32) + +// A basic function to be used as both initializer/deinitializer +orc::shared::CWrapperFunctionResult incrementWrapper(const char *ArgData, + size_t ArgSize) { + return WrapperFunction::handle( + ArgData, ArgSize, + [](ExecutorAddr A) -> Error { + *A.toPtr() += 1; + return Error::success(); + }) + .release(); +} + +TEST(SharedMemoryMapperTest, MemReserveInitializeDeinitializeRelease) { + // These counters are used to track how many times the initializer and + // deinitializer functions are called + int InitializeCounter = 0; + int DeinitializeCounter = 0; + + auto SelfEPC = cantFail(SelfExecutorProcessControl::Create()); + + ExecutorSharedMemoryMapperService MapperService; + + SharedMemoryMapper::SymbolAddrs SAs; + { + StringMap Map; + MapperService.addBootstrapSymbols(Map); + SAs.Instance = Map[rt::ExecutorSharedMemoryMapperServiceInstanceName]; + SAs.Reserve = Map[rt::ExecutorSharedMemoryMapperServiceReserveWrapperName]; + SAs.Initialize = + Map[rt::ExecutorSharedMemoryMapperServiceInitializeWrapperName]; + SAs.Deinitialize = + Map[rt::ExecutorSharedMemoryMapperServiceDeinitializeWrapperName]; + SAs.Release = Map[rt::ExecutorSharedMemoryMapperServiceReleaseWrapperName]; + } + + std::string TestString = "Hello, World!"; + + // barrier + std::promise P; + auto F = P.get_future(); + + { + auto PageSize = cantFail(sys::Process::getPageSize()); + size_t ReqSize = PageSize; + + std::unique_ptr Mapper = + std::make_unique(*SelfEPC, SAs); + + Mapper->reserve(ReqSize, [&](Expected Result) { + EXPECT_THAT_ERROR(Result.takeError(), Succeeded()); + auto Reservation = std::move(*Result); + { + char *Addr = Mapper->prepare(Reservation.Start, TestString.size() + 1); + std::strcpy(Addr, TestString.c_str()); + } + MemoryMapper::AllocInfo AI; + { + MemoryMapper::AllocInfo::SegInfo SI; + SI.Offset = 0; + SI.ContentSize = TestString.size() + 1; + SI.ZeroFillSize = PageSize - SI.ContentSize; + SI.Prot = sys::Memory::MF_READ | sys::Memory::MF_WRITE; + + AI.MappingBase = Reservation.Start; + AI.Segments.push_back(SI); + AI.Actions.push_back( + {cantFail(WrapperFunctionCall::Create>( + ExecutorAddr::fromPtr(incrementWrapper), + ExecutorAddr::fromPtr(&InitializeCounter))), + cantFail(WrapperFunctionCall::Create>( + ExecutorAddr::fromPtr(incrementWrapper), + ExecutorAddr::fromPtr(&DeinitializeCounter)))}); + } + + EXPECT_EQ(InitializeCounter, 0); + EXPECT_EQ(DeinitializeCounter, 0); + + Mapper->initialize(AI, [&, Reservation](Expected Result) { + EXPECT_THAT_ERROR(Result.takeError(), Succeeded()); + + EXPECT_EQ(TestString, std::string(static_cast( + Reservation.Start.toPtr()))); + + EXPECT_EQ(InitializeCounter, 1); + EXPECT_EQ(DeinitializeCounter, 0); + + Mapper->deinitialize({*Result}, [&, Reservation](Error Err) { + EXPECT_THAT_ERROR(std::move(Err), Succeeded()); + + EXPECT_EQ(InitializeCounter, 1); + EXPECT_EQ(DeinitializeCounter, 1); + + Mapper->release({Reservation.Start}, [&](Error Err) { + EXPECT_THAT_ERROR(std::move(Err), Succeeded()); + + P.set_value(); + }); + }); + }); + }); + + // This will block the test if any of the above callbacks are not executed + F.wait(); + // Mapper must be destructed before calling shutdown to avoid double free + } + + EXPECT_THAT_ERROR(MapperService.shutdown(), Succeeded()); + cantFail(SelfEPC->disconnect()); +} + +#endif