diff --git a/llvm/include/llvm/ExecutionEngine/Orc/MemoryMapper.h b/llvm/include/llvm/ExecutionEngine/Orc/MemoryMapper.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/Orc/MemoryMapper.h @@ -0,0 +1,82 @@ +//===- MemoryMapper.h - Cross-process memory mapper -------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// TODO +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_MEMORYMAPPER_H +#define LLVM_EXECUTIONENGINE_ORC_MEMORYMAPPER_H + +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/Support/Error.h" + +namespace llvm { +namespace orc { + +class MemoryMapper { +public: + struct Mapping { + ExecutorAddr RemoteAddr; + void *LocalAddr; + size_t Size; + + operator ExecutorAddr() const { return RemoteAddr; } + }; + + using ReserveResult = Expected; + using OnReservedFunction = unique_function; + virtual void reserve(size_t NumBytes, OnReservedFunction OnReserved) = 0; + virtual ReserveResult reserve(size_t NumBytes); + + using OnFinalizedFunction = unique_function; + virtual void finalize(Mapping &M, + tpctypes::SharedMemoryFinalizeRequest &Actions, + OnFinalizedFunction OnReserved) = 0; + virtual Error finalize(Mapping &M, + tpctypes::SharedMemoryFinalizeRequest &Actions); + + using OnDeallocatedFunction = unique_function; + virtual Error release(std::vector &Allocs, + OnDeallocatedFunction OnDeallocation) = 0; + virtual Error release(std::vector &Allocs); + + virtual ~MemoryMapper(); +}; + +class SharedMemoryMapper final : public MemoryMapper { +public: + struct SymbolAddrs { + ExecutorAddr Manager; + ExecutorAddr Reserve; + ExecutorAddr Finalize; + ExecutorAddr Deallocate; + }; + + SharedMemoryMapper(ExecutorProcessControl &EPC, SymbolAddrs SAs) + : EPC(EPC), SAs(SAs) {} + + void reserve(size_t NumBytes, OnReservedFunction OnReserved) override; + + void finalize(Mapping &M, tpctypes::SharedMemoryFinalizeRequest &Actions, + OnFinalizedFunction OnReserved) override; + + Error release(std::vector &Mappings, + OnDeallocatedFunction OnDeallocated) override; + + ~SharedMemoryMapper() override; + +private: + ExecutorProcessControl &EPC; + SymbolAddrs SAs; +}; + +} // namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_MEMORYMAPPER_H diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h --- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h @@ -31,6 +31,11 @@ extern const char *SimpleExecutorMemoryManagerFinalizeWrapperName; extern const char *SimpleExecutorMemoryManagerDeallocateWrapperName; +extern const char *ExecutorSharedMemoryManagerInstanceName; +extern const char *ExecutorSharedMemoryManagerReserveWrapperName; +extern const char *ExecutorSharedMemoryManagerFinalizeWrapperName; +extern const char *ExecutorSharedMemoryManagerDeallocateWrapperName; + extern const char *MemoryWriteUInt8sWrapperName; extern const char *MemoryWriteUInt16sWrapperName; extern const char *MemoryWriteUInt32sWrapperName; @@ -58,6 +63,14 @@ using SPSSimpleExecutorMemoryManagerDeallocateSignature = shared::SPSError( shared::SPSExecutorAddr, shared::SPSSequence); +using SPSExecutorSharedMemoryManagerReserveSignature = shared::SPSExpected< + shared::SPSTuple>( + shared::SPSExecutorAddr, uint64_t); +using SPSExecutorSharedMemoryManagerFinalizeSignature = shared::SPSError( + shared::SPSExecutorAddr, shared::SPSSharedMemoryFinalizeRequest); +using SPSExecutorSharedMemoryManagerDeallocateSignature = shared::SPSError( + shared::SPSExecutorAddr, shared::SPSSequence); + using SPSRunAsMainSignature = int64_t(shared::SPSExecutorAddr, shared::SPSSequence); diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h --- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h @@ -82,6 +82,17 @@ shared::AllocActions Actions; }; +struct SharedMemorySegFinalizeRequest { + WireProtectionFlags Prot; + ExecutorAddr Addr; + uint64_t Size; +}; + +struct SharedMemoryFinalizeRequest { + std::vector Segments; + shared::AllocActions Actions; +}; + template struct UIntWrite { UIntWrite() = default; UIntWrite(ExecutorAddr Addr, T Value) : Addr(Addr), Value(Value) {} @@ -131,6 +142,13 @@ using SPSFinalizeRequest = SPSTuple, SPSSequence>; +using SPSSharedMemorySegFinalizeRequest = + SPSTuple; + +using SPSSharedMemoryFinalizeRequest = + SPSTuple, + SPSSequence>; + template using SPSMemoryAccessUIntWrite = SPSTuple; @@ -204,6 +222,49 @@ } }; +template <> +class SPSSerializationTraits { + using SFRAL = SPSSharedMemorySegFinalizeRequest::AsArgList; + +public: + static size_t size(const tpctypes::SharedMemorySegFinalizeRequest &SFR) { + return SFRAL::size(SFR.Prot, SFR.Addr, SFR.Size); + } + + static bool serialize(SPSOutputBuffer &OB, + const tpctypes::SharedMemorySegFinalizeRequest &SFR) { + + return SFRAL::serialize(OB, SFR.Prot, SFR.Addr, SFR.Size); + } + + static bool deserialize(SPSInputBuffer &IB, + tpctypes::SharedMemorySegFinalizeRequest &SFR) { + return SFRAL::deserialize(IB, SFR.Prot, SFR.Addr, SFR.Size); + } +}; + +template <> +class SPSSerializationTraits { + using FRAL = SPSSharedMemoryFinalizeRequest::AsArgList; + +public: + static size_t size(const tpctypes::SharedMemoryFinalizeRequest &FR) { + return FRAL::size(FR.Segments, FR.Actions); + } + + static bool serialize(SPSOutputBuffer &OB, + const tpctypes::SharedMemoryFinalizeRequest &FR) { + return FRAL::serialize(OB, FR.Segments, FR.Actions); + } + + static bool deserialize(SPSInputBuffer &IB, + tpctypes::SharedMemoryFinalizeRequest &FR) { + return FRAL::deserialize(IB, FR.Segments, FR.Actions); + } +}; + template class SPSSerializationTraits, tpctypes::UIntWrite> { @@ -244,7 +305,6 @@ } }; - } // end namespace shared } // end namespace orc } // end namespace llvm diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryManager.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryManager.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryManager.h @@ -0,0 +1,67 @@ +//===---------------- ExecutorSharedMemoryManager.h -------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// TODO +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_EXECUTORSHAREDMEMORYMANAGER_H +#define LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_EXECUTORSHAREDMEMORYMANAGER_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h" +#include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorBootstrapService.h" +#include "llvm/Support/Error.h" + +#include +#include + +namespace llvm { +namespace orc { +namespace rt_bootstrap { + +class ExecutorSharedMemoryManager : public ExecutorBootstrapService { +public: + virtual ~ExecutorSharedMemoryManager(); + + Expected> allocate(uint64_t Size); + Error finalize(tpctypes::SharedMemoryFinalizeRequest &FR); + Error deallocate(const std::vector &Bases); + + Error shutdown() override; + void addBootstrapSymbols(StringMap &M) override; + +private: + struct Allocation { + size_t Size = 0; + std::vector DeallocationActions; + }; + + using AllocationsMap = DenseMap; + + Error deallocateImpl(void *Base, Allocation &A); + + static llvm::orc::shared::CWrapperFunctionResult + reserveWrapper(const char *ArgData, size_t ArgSize); + + static llvm::orc::shared::CWrapperFunctionResult + finalizeWrapper(const char *ArgData, size_t ArgSize); + + static llvm::orc::shared::CWrapperFunctionResult + deallocateWrapper(const char *ArgData, size_t ArgSize); + + std::atomic SharedMemoryCount{0}; + std::mutex M; + AllocationsMap Allocations; +}; + +} // end namespace rt_bootstrap +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_EXECUTORSHAREDMEMORYMANAGER_H diff --git a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt --- a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt +++ b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt @@ -23,6 +23,7 @@ LookupAndRecordAddrs.cpp LLJIT.cpp MachOPlatform.cpp + MemoryMapper.cpp ELFNixPlatform.cpp Mangling.cpp ObjectLinkingLayer.cpp diff --git a/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp b/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp @@ -0,0 +1,144 @@ +//===- MemoryMapper.cpp - Cross-process memory mapper ------------*- C++ -*-==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// TODO +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/MemoryMapper.h" +#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" + +#include + +#ifdef LLVM_ON_UNIX +#include +#include +#include +#endif + +namespace llvm { +namespace orc { + +MemoryMapper::ReserveResult MemoryMapper::reserve(size_t NumBytes) { + std::promise P; + auto F = P.get_future(); + reserve(NumBytes, + [&](MemoryMapper::ReserveResult R) { P.set_value(std::move(R)); }); + return F.get(); +} + +Error MemoryMapper::finalize(MemoryMapper::Mapping &M, + tpctypes::SharedMemoryFinalizeRequest &Actions) { + std::promise P; + auto F = P.get_future(); + finalize(M, Actions, [&](Error R) { P.set_value(std::move(R)); }); + return F.get(); +} + +Error MemoryMapper::release(std::vector &Allocs) { + std::promise P; + auto F = P.get_future(); + Error ErrLocal = release(Allocs, [&](Error R) { P.set_value(std::move(R)); }); + return joinErrors(std::move(ErrLocal), F.get()); +} + +MemoryMapper::~MemoryMapper() {} + +void SharedMemoryMapper::reserve(size_t Size, OnReservedFunction OnReserved) { + EPC.callSPSWrapperAsync( + SAs.Reserve, + [Size, OnReserved = std::move(OnReserved)]( + Error SerializationErr, + Expected> Result) mutable { + if (SerializationErr) { + cantFail(Result.takeError()); + return OnReserved(std::move(SerializationErr)); + } + if (!Result) + return OnReserved(Result.takeError()); + + ExecutorAddr RemoteAddr; + std::string SharedMemoryName; + std::tie(RemoteAddr, SharedMemoryName) = std::move(*Result); + +#ifdef LLVM_ON_UNIX + int SharedMemoryFile = shm_open(SharedMemoryName.c_str(), O_RDWR, 0700); + if (SharedMemoryFile < 0) { + return OnReserved(errorCodeToError( + std::error_code(errno, std::generic_category()))); + } + + // this prevents other programs from accessing it by name + shm_unlink(SharedMemoryName.c_str()); + + void *LocalAddr = mmap(nullptr, Size, PROT_READ | PROT_WRITE, + MAP_SHARED, SharedMemoryFile, 0); + if (LocalAddr == MAP_FAILED) { + return OnReserved(errorCodeToError( + std::error_code(errno, std::generic_category()))); + } + close(SharedMemoryFile); +#endif + OnReserved(Mapping{RemoteAddr, LocalAddr, Size}); + }, + SAs.Manager, Size); +} + +void SharedMemoryMapper::finalize(Mapping &M, + tpctypes::SharedMemoryFinalizeRequest &FR, + OnFinalizedFunction OnFinalize) { + EPC.callSPSWrapperAsync( + SAs.Finalize, + [OnFinalize = std::move(OnFinalize)](Error SerializationErr, + Error FinalizeErr) mutable { + if (SerializationErr) { + cantFail(std::move(FinalizeErr)); + OnFinalize(std::move(SerializationErr)); + } else if (FinalizeErr) + OnFinalize(std::move(FinalizeErr)); + else + OnFinalize(Error::success()); + }, + SAs.Manager, std::move(FR)); +} + +Error SharedMemoryMapper::release(std::vector &Mappings, + OnDeallocatedFunction OnDeallocated) { + std::vector Bases(Mappings.begin(), Mappings.end()); + EPC.callSPSWrapperAsync< + rt::SPSExecutorSharedMemoryManagerDeallocateSignature>( + SAs.Deallocate, + [OnDeallocated = std::move(OnDeallocated)](Error SerErr, + Error DeallocErr) mutable { + if (SerErr) { + cantFail(std::move(DeallocErr)); + OnDeallocated(std::move(SerErr)); + } else + OnDeallocated(std::move(DeallocErr)); + }, + SAs.Manager, Bases); + + Error Err = Error::success(); + + for (auto M : Mappings) { +#ifdef LLVM_ON_UNIX + if (munmap(M.LocalAddr, M.Size) != 0) { + Err = joinErrors(std::move(Err), errorCodeToError(std::error_code( + errno, std::generic_category()))); + } +#endif + } + + return Err; +} + +SharedMemoryMapper::~SharedMemoryMapper() {} + +} // namespace orc + +} // namespace llvm diff --git a/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp b/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp --- a/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp +++ b/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp @@ -26,6 +26,14 @@ "__llvm_orc_SimpleExecutorMemoryManager_finalize_wrapper"; const char *SimpleExecutorMemoryManagerDeallocateWrapperName = "__llvm_orc_SimpleExecutorMemoryManager_deallocate_wrapper"; +const char *ExecutorSharedMemoryManagerInstanceName = + "__llvm_orc_ExecutorSharedMemoryManager_Instance"; +const char *ExecutorSharedMemoryManagerReserveWrapperName = + "__llvm_orc_ExecutorSharedMemoryManager_reserve_wrapper"; +const char *ExecutorSharedMemoryManagerFinalizeWrapperName = + "__llvm_orc_ExecutorSharedMemoryManager_finalize_wrapper"; +const char *ExecutorSharedMemoryManagerDeallocateWrapperName = + "__llvm_orc_ExecutorSharedMemoryManager_deallocate_wrapper"; const char *MemoryWriteUInt8sWrapperName = "__llvm_orc_bootstrap_mem_write_uint8s_wrapper"; const char *MemoryWriteUInt16sWrapperName = diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt --- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt +++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt @@ -1,4 +1,5 @@ add_llvm_component_library(LLVMOrcTargetProcess + ExecutorSharedMemoryManager.cpp JITLoaderGDB.cpp OrcRTBootstrap.cpp RegisterEHFrames.cpp diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryManager.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryManager.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryManager.cpp @@ -0,0 +1,297 @@ +//===- ExecutorShardMemoryManagare.cpp - Executor-side shared memory mgmt -===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryManager.h" + +#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/Process.h" + +#include + +#ifdef LLVM_ON_UNIX +#include +#include +#include +#include +#endif + +namespace llvm { +namespace orc { +namespace rt_bootstrap { + +ExecutorSharedMemoryManager::~ExecutorSharedMemoryManager() { + assert(Allocations.empty() && "shutdown not called?"); +} + +Expected> +ExecutorSharedMemoryManager::allocate(uint64_t Size) { +#ifdef LLVM_ON_UNIX + std::string SharedMemoryName; + { + std::stringstream SharedMemoryNameStream; + SharedMemoryNameStream << "/jitlink_" << sys::Process::getProcessId() << '_' + << (++SharedMemoryCount); + SharedMemoryName = SharedMemoryNameStream.str(); + } + + int SharedMemoryFile = + shm_open(SharedMemoryName.c_str(), O_RDWR | O_CREAT | O_EXCL, 0700); + if (SharedMemoryFile < 0) { + return errorCodeToError(std::error_code(errno, std::generic_category())); + } + + // by default size is 0 + if (ftruncate(SharedMemoryFile, Size) < 0) { + return errorCodeToError(std::error_code(errno, std::generic_category())); + } + + void *Addr = mmap(nullptr, Size, PROT_NONE, MAP_SHARED, SharedMemoryFile, 0); + if (Addr == MAP_FAILED) { + return errorCodeToError(std::error_code(errno, std::generic_category())); + } + + close(SharedMemoryFile); + + { + std::lock_guard Lock(M); + assert(!Allocations.count(Addr) && "Duplicate allocation addr"); + Allocations[Addr].Size = Size; + } + + return std::make_pair(ExecutorAddr::fromPtr(Addr), + std::move(SharedMemoryName)); +#endif +} + +Error ExecutorSharedMemoryManager::finalize( + tpctypes::SharedMemoryFinalizeRequest &FR) { + ExecutorAddr Base(~0ULL); + std::vector DeallocationActions; + size_t SuccessfulFinalizationActions = 0; + + if (FR.Segments.empty()) { + if (FR.Actions.empty()) + return Error::success(); + return make_error( + "Finalization actions attached to empty finalization request", + inconvertibleErrorCode()); + } + + for (auto &Seg : FR.Segments) + Base = std::min(Base, Seg.Addr); + + for (auto &ActPair : FR.Actions) + if (ActPair.Dealloc) + DeallocationActions.push_back(ActPair.Dealloc); + + // Get the Allocation for this finalization. + size_t AllocSize = 0; + { + std::lock_guard Lock(M); + auto I = Allocations.find(Base.toPtr()); + if (I == Allocations.end()) + return make_error( + formatv("Attempt to finalize unrecognized allocation {0:x}", + Base.getValue()), + inconvertibleErrorCode()); + AllocSize = I->second.Size; + I->second.DeallocationActions = std::move(DeallocationActions); + } + ExecutorAddr AllocEnd = Base + ExecutorAddrDiff(AllocSize); + + // Bail-out function: this will run deallocation actions corresponding to any + // completed finalization actions, then deallocate memory. + auto BailOut = [&](Error Err) { + std::pair AllocToDestroy; + + // Get allocation to destory. + { + std::lock_guard Lock(M); + auto I = Allocations.find(Base.toPtr()); + + // Check for missing allocation (effective a double free). + if (I == Allocations.end()) + return joinErrors( + std::move(Err), + make_error( + formatv("No allocation entry found for {0:x}", Base.getValue()), + inconvertibleErrorCode())); + AllocToDestroy = std::move(*I); + Allocations.erase(I); + } + + // Run deallocation actions for all completed finalization actions. + while (SuccessfulFinalizationActions) + Err = + joinErrors(std::move(Err), FR.Actions[--SuccessfulFinalizationActions] + .Dealloc.runWithSPSRetErrorMerged()); + + // Deallocate memory. + // TODO + sys::MemoryBlock MB(AllocToDestroy.first, AllocToDestroy.second.Size); + if (auto EC = sys::Memory::releaseMappedMemory(MB)) + Err = joinErrors(std::move(Err), errorCodeToError(EC)); + + return Err; + }; + + // Copy content and apply permissions. + for (auto &Seg : FR.Segments) { + + ExecutorAddr SegEnd = Seg.Addr + ExecutorAddrDiff(Seg.Size); + if (LLVM_UNLIKELY(Seg.Addr < Base || SegEnd > AllocEnd)) + return BailOut(make_error( + formatv("Segment {0:x} -- {1:x} crosses boundary of " + "allocation {2:x} -- {3:x}", + Seg.Addr.getValue(), SegEnd.getValue(), Base.getValue(), + AllocEnd.getValue()), + inconvertibleErrorCode())); + + char *Mem = Seg.Addr.toPtr(); + assert(Seg.Size <= std::numeric_limits::max()); + +#ifdef LLVM_ON_UNIX + + int MmapProt = 0; + if (Seg.Prot & tpctypes::WPF_Read) + MmapProt |= PROT_READ; + if (Seg.Prot & tpctypes::WPF_Write) + MmapProt |= PROT_WRITE; + if (Seg.Prot & tpctypes::WPF_Exec) + MmapProt |= PROT_EXEC; + mprotect(Mem, Seg.Size, MmapProt); + + if (Seg.Prot & tpctypes::WPF_Exec) + sys::Memory::InvalidateInstructionCache(Mem, Seg.Size); + } +#endif + + // Run finalization actions. + for (auto &ActPair : FR.Actions) { + if (auto Err = ActPair.Finalize.runWithSPSRetErrorMerged()) + return BailOut(std::move(Err)); + ++SuccessfulFinalizationActions; + } + + return Error::success(); +} + +Error ExecutorSharedMemoryManager::deallocate( + const std::vector &Bases) { + std::vector> AllocPairs; + AllocPairs.reserve(Bases.size()); + + // Get allocation to destory. + Error Err = Error::success(); + { + std::lock_guard Lock(M); + for (auto &Base : Bases) { + auto I = Allocations.find(Base.toPtr()); + + // Check for missing allocation (effective a double free). + if (I != Allocations.end()) { + AllocPairs.push_back(std::move(*I)); + Allocations.erase(I); + } else + Err = joinErrors( + std::move(Err), + make_error( + formatv("No allocation entry found for {0:x}", Base.getValue()), + inconvertibleErrorCode())); + } + } + + while (!AllocPairs.empty()) { + auto &P = AllocPairs.back(); + Err = joinErrors(std::move(Err), deallocateImpl(P.first, P.second)); + AllocPairs.pop_back(); + } + + return Err; +} + +Error ExecutorSharedMemoryManager::shutdown() { + + AllocationsMap AM; + { + std::lock_guard Lock(M); + AM = std::move(Allocations); + } + + Error Err = Error::success(); + for (auto &KV : AM) + Err = joinErrors(std::move(Err), deallocateImpl(KV.first, KV.second)); + return Err; +} + +void ExecutorSharedMemoryManager::addBootstrapSymbols( + StringMap &M) { + M[rt::ExecutorSharedMemoryManagerInstanceName] = ExecutorAddr::fromPtr(this); + M[rt::ExecutorSharedMemoryManagerReserveWrapperName] = + ExecutorAddr::fromPtr(&reserveWrapper); + M[rt::ExecutorSharedMemoryManagerFinalizeWrapperName] = + ExecutorAddr::fromPtr(&finalizeWrapper); + M[rt::ExecutorSharedMemoryManagerDeallocateWrapperName] = + ExecutorAddr::fromPtr(&deallocateWrapper); +} + +Error ExecutorSharedMemoryManager::deallocateImpl(void *Base, Allocation &A) { + Error Err = Error::success(); + + while (!A.DeallocationActions.empty()) { + Err = joinErrors(std::move(Err), + A.DeallocationActions.back().runWithSPSRetErrorMerged()); + A.DeallocationActions.pop_back(); + } + +#ifdef LLVM_ON_UNIX + if (munmap(Base, A.Size) != 0) + Err = joinErrors(std::move(Err), errorCodeToError(std::error_code( + errno, std::generic_category()))); +#endif + + return Err; +} + +llvm::orc::shared::CWrapperFunctionResult +ExecutorSharedMemoryManager::reserveWrapper(const char *ArgData, + size_t ArgSize) { + return shared::WrapperFunction< + rt::SPSExecutorSharedMemoryManagerReserveSignature>:: + handle(ArgData, ArgSize, + shared::makeMethodWrapperHandler( + &ExecutorSharedMemoryManager::allocate)) + .release(); +} + +llvm::orc::shared::CWrapperFunctionResult +ExecutorSharedMemoryManager::finalizeWrapper(const char *ArgData, + size_t ArgSize) { + return shared::WrapperFunction< + rt::SPSExecutorSharedMemoryManagerFinalizeSignature>:: + handle(ArgData, ArgSize, + shared::makeMethodWrapperHandler( + &ExecutorSharedMemoryManager::finalize)) + .release(); +} + +llvm::orc::shared::CWrapperFunctionResult +ExecutorSharedMemoryManager::deallocateWrapper(const char *ArgData, + size_t ArgSize) { + return shared::WrapperFunction< + rt::SPSExecutorSharedMemoryManagerDeallocateSignature>:: + handle(ArgData, ArgSize, + shared::makeMethodWrapperHandler( + &ExecutorSharedMemoryManager::deallocate)) + .release(); +} + +} // namespace rt_bootstrap +} // end namespace orc +} // end namespace llvm diff --git a/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt b/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt --- a/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt +++ b/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt @@ -18,12 +18,14 @@ CoreAPIsTest.cpp ExecutorAddressTest.cpp ExecutionSessionWrapperFunctionCallsTest.cpp + ExecutorSharedMemoryManagerTest.cpp EPCGenericJITLinkMemoryManagerTest.cpp EPCGenericMemoryAccessTest.cpp IndirectionUtilsTest.cpp JITTargetMachineBuilderTest.cpp LazyCallThroughAndReexportsTest.cpp LookupAndRecordAddrsTest.cpp + SharedMemoryMapperTest.cpp ObjectLinkingLayerTest.cpp OrcCAPITest.cpp OrcTestCommon.cpp diff --git a/llvm/unittests/ExecutionEngine/Orc/ExecutorSharedMemoryManagerTest.cpp b/llvm/unittests/ExecutionEngine/Orc/ExecutorSharedMemoryManagerTest.cpp new file mode 100755 --- /dev/null +++ b/llvm/unittests/ExecutionEngine/Orc/ExecutorSharedMemoryManagerTest.cpp @@ -0,0 +1,104 @@ +//===---------------- ExecutorSharedMemoryManagerTest.cpp -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryManager.h" +#include "llvm/Testing/Support/Error.h" +#include "gtest/gtest.h" + +#include +#include +#include + +#ifdef LLVM_ON_UNIX +#include +#include +#endif + +using namespace llvm; +using namespace llvm::orc; +using namespace llvm::orc::shared; +using namespace llvm::orc::rt_bootstrap; + +namespace { + +#ifdef LLVM_ON_UNIX +orc::shared::CWrapperFunctionResult incrementWrapper(const char *ArgData, + size_t ArgSize) { + return WrapperFunction::handle( + ArgData, ArgSize, + [](ExecutorAddr A) -> Error { + *A.toPtr() += 1; + return Error::success(); + }) + .release(); +} + +TEST(ExecutorSharedMemoryManagerTest, AllocFinalizeFree) { + ExecutorSharedMemoryManager MemMgr; + + constexpr unsigned AllocSize = 16384; + + auto Mem = MemMgr.allocate(AllocSize); + EXPECT_THAT_ERROR(Mem.takeError(), Succeeded()); + + ExecutorAddr ExecAddr; + std::string Name; + std::tie(ExecAddr, Name) = *Mem; + + int File = shm_open(Name.c_str(), O_RDWR, 0700); + EXPECT_GT(File, 0); + + void *WorkingAddr = + mmap(nullptr, AllocSize, PROT_READ | PROT_WRITE, MAP_SHARED, File, 0); + EXPECT_NE(WorkingAddr, MAP_FAILED); + + shm_unlink(Name.c_str()); + + std::string HW = "Hello, world!"; + + std::strcpy((char *)WorkingAddr, HW.c_str()); + + int FinalizeCounter = 0; + int DeallocateCounter = 0; + + tpctypes::SharedMemoryFinalizeRequest FR; + FR.Segments.push_back(tpctypes::SharedMemorySegFinalizeRequest{ + tpctypes::WPF_Read | tpctypes::WPF_Write, + ExecAddr, + AllocSize, + }); + FR.Actions.push_back( + {/* Finalize: */ + cantFail(WrapperFunctionCall::Create>( + ExecutorAddr::fromPtr(incrementWrapper), + ExecutorAddr::fromPtr(&FinalizeCounter))), + /* Deallocate: */ + cantFail(WrapperFunctionCall::Create>( + ExecutorAddr::fromPtr(incrementWrapper), + ExecutorAddr::fromPtr(&DeallocateCounter)))}); + + EXPECT_EQ(FinalizeCounter, 0); + EXPECT_EQ(DeallocateCounter, 0); + + auto FinalizeErr = MemMgr.finalize(FR); + EXPECT_THAT_ERROR(std::move(FinalizeErr), Succeeded()); + + EXPECT_EQ(FinalizeCounter, 1); + EXPECT_EQ(DeallocateCounter, 0); + + EXPECT_EQ(HW, std::string(ExecAddr.toPtr())); + + auto DeallocateErr = MemMgr.deallocate({ExecAddr}); + EXPECT_THAT_ERROR(std::move(DeallocateErr), Succeeded()); + + EXPECT_EQ(FinalizeCounter, 1); + EXPECT_EQ(DeallocateCounter, 1); +} +#endif + +} // namespace diff --git a/llvm/unittests/ExecutionEngine/Orc/SharedMemoryMapperTest.cpp b/llvm/unittests/ExecutionEngine/Orc/SharedMemoryMapperTest.cpp new file mode 100644 --- /dev/null +++ b/llvm/unittests/ExecutionEngine/Orc/SharedMemoryMapperTest.cpp @@ -0,0 +1,79 @@ +//===- SharedMemoryMapperTest.cpp -- Tests for SharedMemoryMapper ---------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "OrcTestCommon.h" + +#include "llvm/ExecutionEngine/Orc/MemoryMapper.h" +#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" +#include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryManager.h" +#include "llvm/Testing/Support/Error.h" + +using namespace llvm; +using namespace llvm::orc; +using namespace llvm::orc::shared; +using namespace llvm::orc::rt_bootstrap; + +TEST(SharedMemoryMapperTest, MemReserveFinalizeDealloc) { + auto SelfEPC = cantFail(SelfExecutorProcessControl::Create()); + + ExecutorSharedMemoryManager MemMgr; + + SharedMemoryMapper::SymbolAddrs SAs; + { + StringMap Map; + MemMgr.addBootstrapSymbols(Map); + SAs.Manager = Map[rt::ExecutorSharedMemoryManagerInstanceName]; + SAs.Reserve = Map[rt::ExecutorSharedMemoryManagerReserveWrapperName]; + SAs.Finalize = Map[rt::ExecutorSharedMemoryManagerFinalizeWrapperName]; + SAs.Deallocate = Map[rt::ExecutorSharedMemoryManagerDeallocateWrapperName]; + } + + // barrier + std::promise RP; + auto RF = RP.get_future(); + + { + size_t ReqSize = 4096 * 4; + auto Mapper = std::make_unique(*SelfEPC, SAs); + Mapper->reserve(ReqSize, [Mapper = std::move(Mapper), RP = std::move(RP), + ReqSize]( + Expected M) mutable { + EXPECT_THAT_ERROR(M.takeError(), Succeeded()); + EXPECT_GE(M->Size, ReqSize); + + tpctypes::SharedMemoryFinalizeRequest FR; + FR.Segments.push_back(tpctypes::SharedMemorySegFinalizeRequest{ + tpctypes::WPF_Read | tpctypes::WPF_Write, + M->RemoteAddr, + ReqSize, + }); + + Mapper->finalize(*M, FR, + [Mapper = std::move(Mapper), RP = std::move(RP), + M = std::move(*M)](Error Err) mutable { + EXPECT_THAT_ERROR(std::move(Err), Succeeded()); + + std::vector Blocks = {M}; + auto Res = Mapper->release( + Blocks, [RP = std::move(RP)](Error Err) mutable { + EXPECT_THAT_ERROR(std::move(Err), Succeeded()); + RP.set_value(); + }); + + // Unmapping from controller process succeeded + EXPECT_THAT_ERROR(std::move(Res), Succeeded()); + }); + }); + } + + RF.wait(); + + EXPECT_THAT_ERROR(MemMgr.shutdown(), Succeeded()); + + cantFail(SelfEPC->disconnect()); +}