diff --git a/llvm/include/llvm/ExecutionEngine/Orc/MemoryMapper.h b/llvm/include/llvm/ExecutionEngine/Orc/MemoryMapper.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/Orc/MemoryMapper.h @@ -0,0 +1,141 @@ +//===- MemoryMapper.h - Cross-process memory mapper -------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Cross-process (and in-process) memory mapping and transfer +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_MEMORYMAPPER_H +#define LLVM_EXECUTIONENGINE_ORC_MEMORYMAPPER_H + +#include "llvm/ExecutionEngine/Orc/Core.h" + +#include + +namespace llvm { +namespace orc { + +/// Manages mapping, content transfer and protections for JIT memory +class MemoryMapper { +public: + /// Represents a single allocation containing multiple segments and + /// initialization and deinitialization actions + struct AllocInfo { + struct SegInfo { + ExecutorAddrDiff Offset; + const char *WorkingMem; + size_t ContentSize; + size_t ZeroFillSize; + unsigned Prot; + }; + + ExecutorAddr MappingBase; + std::vector Segments; + shared::AllocActions Actions; + }; + + using OnReservedFunction = unique_function)>; + + /// Reserves address space in executor process + virtual void reserve(size_t NumBytes, OnReservedFunction OnReserved) = 0; + + /// Provides working memory + virtual Expected prepare(ExecutorAddr Addr, size_t ContentSize, + unsigned Protections) = 0; + + using OnInitializedFunction = unique_function)>; + + /// Ensures executor memory is synchronized with working copy memory, sends + /// functions to be called after initilization and before deinitialization and + /// applies memory protections + virtual void initialize(AllocInfo &AI, + OnInitializedFunction OnInitialized) = 0; + + using OnDeinitializedFunction = unique_function; + + /// Runs previously specified denitialization actions + virtual void deinitialize(std::vector &Allocations, + OnDeinitializedFunction OnDeInitialized) = 0; + + using OnReleasedFunction = unique_function; + + /// Unmaps address space from both process + virtual void release(std::vector &Reservations, + OnReleasedFunction OnRelease) = 0; + + Expected reserve(size_t NumBytes) { + std::promise> P; + auto F = P.get_future(); + reserve(NumBytes, [&](auto R) { P.set_value(std::move(R)); }); + return F.get(); + } + + Expected initialize(AllocInfo &AI) { + std::promise> P; + auto F = P.get_future(); + initialize(AI, [&](auto R) { P.set_value(std::move(R)); }); + return F.get(); + } + + Error deinitialize(std::vector &Allocations) { + std::promise P; + auto F = P.get_future(); + deinitialize(Allocations, [&](auto R) { P.set_value(std::move(R)); }); + return F.get(); + } + + Error release(std::vector &Reservations) { + std::promise P; + auto F = P.get_future(); + release(Reservations, [&](auto R) { P.set_value(std::move(R)); }); + return F.get(); + } + + virtual ~MemoryMapper(); +}; + +class InProcessMemoryMapper final : public MemoryMapper { +public: + InProcessMemoryMapper() {} + + using MemoryMapper::reserve; + void reserve(size_t NumBytes, OnReservedFunction OnReserved) override; + + using MemoryMapper::initialize; + void initialize(AllocInfo &AI, OnInitializedFunction OnInitialized) override; + + Expected prepare(ExecutorAddr Addr, size_t ContentSize, + unsigned Protections) override; + + using MemoryMapper::deinitialize; + void deinitialize(std::vector &Allocations, + OnDeinitializedFunction OnDeInitialized) override; + + using MemoryMapper::release; + void release(std::vector &Reservations, + OnReleasedFunction OnRelease) override; + + ~InProcessMemoryMapper() override; + +private: + struct Allocation { + std::vector DeinitializationActions; + }; + using AllocationMap = DenseMap; + + using ReservationMap = DenseMap; + + std::mutex Mutex; + ReservationMap Reservations; + AllocationMap Allocations; +}; + +} // namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_MEMORYMAPPER_H diff --git a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt --- a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt +++ b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt @@ -23,6 +23,7 @@ LookupAndRecordAddrs.cpp LLJIT.cpp MachOPlatform.cpp + MemoryMapper.cpp ELFNixPlatform.cpp Mangling.cpp ObjectLinkingLayer.cpp diff --git a/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp b/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp @@ -0,0 +1,154 @@ +//===- MemoryMapper.cpp - Cross-process memory mapper ------------*- C++ -*-==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/MemoryMapper.h" +#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" + +namespace llvm { +namespace orc { + +MemoryMapper::~MemoryMapper() {} + +void InProcessMemoryMapper::reserve(size_t NumBytes, + OnReservedFunction OnReserved) { + std::error_code EC; + auto MB = sys::Memory::allocateMappedMemory( + NumBytes, nullptr, sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC); + + if (EC) + return OnReserved(errorCodeToError(EC)); + + { + std::lock_guard Lock(Mutex); + Reservations[MB.base()] = MB.allocatedSize(); + } + + OnReserved( + ExecutorAddrRange(ExecutorAddr::fromPtr(MB.base()), MB.allocatedSize())); +} + +Expected InProcessMemoryMapper::prepare(ExecutorAddr Addr, + size_t ContentSize, + unsigned Protections) { + if (auto EC = sys::Memory::protectMappedMemory( + {Addr.toPtr(), ContentSize}, Protections)) { + return errorCodeToError(EC); + } + + return Addr.toPtr(); +} + +void InProcessMemoryMapper::initialize(MemoryMapper::AllocInfo &AI, + OnInitializedFunction OnInitialized) { + ExecutorAddr MinAddr(~0ULL); + + for (auto &Segment : AI.Segments) { + auto Base = AI.MappingBase + Segment.Offset; + auto Size = Segment.ContentSize + Segment.ZeroFillSize; + + if (Base < MinAddr) + MinAddr = Base; + + std::memset( + (AI.MappingBase + Segment.Offset + Segment.ContentSize).toPtr(), + 0, Segment.ZeroFillSize); + + if (auto EC = sys::Memory::protectMappedMemory({Base.toPtr(), Size}, + Segment.Prot)) { + return OnInitialized(errorCodeToError(EC)); + } + if (Segment.Prot & sys::Memory::MF_EXEC) + sys::Memory::InvalidateInstructionCache(Base.toPtr(), Size); + } + { + std::lock_guard Lock(Mutex); + + Allocations[MinAddr].DeinitializationActions.reserve(AI.Actions.size()); + + for (auto &ActionPair : AI.Actions) { + if (auto Err = ActionPair.Finalize.runWithSPSRetErrorMerged()) + return OnInitialized(std::move(Err)); + + Allocations[MinAddr].DeinitializationActions.push_back( + ActionPair.Dealloc); + } + } + OnInitialized(MinAddr); +} + +void InProcessMemoryMapper::deinitialize( + std::vector &Bases, + MemoryMapper::OnDeinitializedFunction OnDeinitialized) { + Error AllErr = Error::success(); + + for (auto Base : Bases) { + bool Failed = false; + + std::lock_guard Lock(Mutex); + + auto &Actions = Allocations[Base].DeinitializationActions; + while (!Actions.empty()) { + auto &Action = Actions.back(); + + if (Error Err = Action.runWithSPSRetErrorMerged()) { + AllErr = joinErrors(std::move(AllErr), std::move(Err)); + Failed = true; + break; + } + Actions.pop_back(); + } + + if (!Failed) + Allocations.erase(Base); + } + + OnDeinitialized(std::move(AllErr)); +} + +void InProcessMemoryMapper::release(std::vector &Bases, + OnReleasedFunction OnReleased) { + Error Err = Error::success(); + + std::lock_guard Lock(Mutex); + + for (auto Base : Bases) { + auto MB = sys::MemoryBlock(Base.toPtr(), + Reservations[Base.toPtr()]); + + auto EC = sys::Memory::releaseMappedMemory(MB); + if (EC) { + Err = joinErrors(std::move(Err), errorCodeToError(EC)); + } + + Reservations.erase(Base.toPtr()); + } + + OnReleased(std::move(Err)); +} + +InProcessMemoryMapper::~InProcessMemoryMapper() { + std::lock_guard Lock(Mutex); + + for (auto &A : Allocations) { + while (!A.second.DeinitializationActions.empty()) { + auto &Action = A.second.DeinitializationActions.back(); + cantFail(Action.runWithSPSRetErrorMerged()); + A.second.DeinitializationActions.pop_back(); + } + } + + for (auto &R : Reservations) { + sys::MemoryBlock MB = {R.first, R.second}; + auto EC = sys::Memory::releaseMappedMemory(MB); + cantFail(errorCodeToError(EC)); + } +} + +} // namespace orc + +} // namespace llvm diff --git a/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt b/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt --- a/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt +++ b/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt @@ -24,6 +24,7 @@ JITTargetMachineBuilderTest.cpp LazyCallThroughAndReexportsTest.cpp LookupAndRecordAddrsTest.cpp + MemoryMapperTest.cpp ObjectLinkingLayerTest.cpp OrcCAPITest.cpp OrcTestCommon.cpp diff --git a/llvm/unittests/ExecutionEngine/Orc/MemoryMapperTest.cpp b/llvm/unittests/ExecutionEngine/Orc/MemoryMapperTest.cpp new file mode 100644 --- /dev/null +++ b/llvm/unittests/ExecutionEngine/Orc/MemoryMapperTest.cpp @@ -0,0 +1,126 @@ +//===------------------------ MemoryMapperTest.cpp ------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/MemoryMapper.h" +#include "llvm/Support/Process.h" +#include "llvm/Testing/Support/Error.h" +#include "gtest/gtest.h" + +using namespace llvm; +using namespace llvm::orc; +using namespace llvm::orc::shared; + +namespace { + +orc::shared::CWrapperFunctionResult incrementWrapper(const char *ArgData, + size_t ArgSize) { + return WrapperFunction::handle( + ArgData, ArgSize, + [](ExecutorAddr A) -> Error { + *A.toPtr() += 1; + return Error::success(); + }) + .release(); +} + +TEST(ExecutorSharedMemoryManagerTest, AllocFinalizeFree) { + int InitializeCounter = 0; + int DeinitializeCounter = 0; + { + std::unique_ptr Mapper = + std::make_unique(); + auto PageSize = sys::Process::getPageSize(); + cantFail(PageSize.takeError()); + auto AllocSize = *PageSize * 2; + + auto Mem = Mapper->reserve(AllocSize); + EXPECT_THAT_ERROR(Mem.takeError(), Succeeded()); + + std::string HW = "Hello, world!"; + + { + auto WA1 = Mapper->prepare(Mem->Start, HW.size() + 1, + sys::Memory::MF_READ | sys::Memory::MF_WRITE); + EXPECT_THAT_ERROR(WA1.takeError(), Succeeded()); + std::strcpy(static_cast(*WA1), HW.c_str()); + } + + MemoryMapper::AllocInfo Alloc1; + { + MemoryMapper::AllocInfo::SegInfo Seg1; + Seg1.Offset = 0; + Seg1.ContentSize = HW.size(); + Seg1.ZeroFillSize = *PageSize - Seg1.ContentSize; + Seg1.Prot = sys::Memory::MF_READ | sys::Memory::MF_WRITE; + + Alloc1.MappingBase = Mem->Start; + Alloc1.Segments.push_back(Seg1); + Alloc1.Actions.push_back( + {cantFail(WrapperFunctionCall::Create>( + ExecutorAddr::fromPtr(incrementWrapper), + ExecutorAddr::fromPtr(&InitializeCounter))), + cantFail(WrapperFunctionCall::Create>( + ExecutorAddr::fromPtr(incrementWrapper), + ExecutorAddr::fromPtr(&DeinitializeCounter)))}); + } + + { + auto WA2 = Mapper->prepare(Mem->Start + *PageSize, HW.size() + 1, + sys::Memory::MF_READ | sys::Memory::MF_WRITE); + EXPECT_THAT_ERROR(WA2.takeError(), Succeeded()); + std::strcpy(static_cast(*WA2), HW.c_str()); + } + + MemoryMapper::AllocInfo Alloc2; + { + MemoryMapper::AllocInfo::SegInfo Seg2; + Seg2.Offset = *PageSize; + Seg2.ContentSize = HW.size(); + Seg2.ZeroFillSize = *PageSize - Seg2.ContentSize; + Seg2.Prot = sys::Memory::MF_READ | sys::Memory::MF_WRITE; + + Alloc2.MappingBase = Mem->Start; + Alloc2.Segments.push_back(Seg2); + Alloc2.Actions.push_back( + {cantFail(WrapperFunctionCall::Create>( + ExecutorAddr::fromPtr(incrementWrapper), + ExecutorAddr::fromPtr(&InitializeCounter))), + cantFail(WrapperFunctionCall::Create>( + ExecutorAddr::fromPtr(incrementWrapper), + ExecutorAddr::fromPtr(&DeinitializeCounter)))}); + } + + EXPECT_EQ(InitializeCounter, 0); + EXPECT_EQ(DeinitializeCounter, 0); + + auto Init1 = Mapper->initialize(Alloc1); + EXPECT_THAT_ERROR(Init1.takeError(), Succeeded()); + EXPECT_EQ(HW, std::string(static_cast(Init1->toPtr()))); + + EXPECT_EQ(InitializeCounter, 1); + EXPECT_EQ(DeinitializeCounter, 0); + + auto Init2 = Mapper->initialize(Alloc2); + EXPECT_THAT_ERROR(Init2.takeError(), Succeeded()); + EXPECT_EQ(HW, std::string(static_cast(Init2->toPtr()))); + + EXPECT_EQ(InitializeCounter, 2); + EXPECT_EQ(DeinitializeCounter, 0); + + std::vector DeinitAddr = {*Init1}; + EXPECT_THAT_ERROR(Mapper->deinitialize(DeinitAddr), Succeeded()); + + EXPECT_EQ(InitializeCounter, 2); + EXPECT_EQ(DeinitializeCounter, 1); + } + + EXPECT_EQ(InitializeCounter, 2); + EXPECT_EQ(DeinitializeCounter, 2); +} + +} // namespace