diff --git a/llvm/include/llvm/ExecutionEngine/Orc/MapperJITLinkMemoryManager.h b/llvm/include/llvm/ExecutionEngine/Orc/MapperJITLinkMemoryManager.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/Orc/MapperJITLinkMemoryManager.h @@ -0,0 +1,56 @@ +//===--------------- MapperJITLinkMemoryManager.h -*- C++ -*---------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Implements JITLinkMemoryManager using MemoryMapper +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_MAPPERJITLINKMEMORYMANAGER_H +#define LLVM_EXECUTIONENGINE_ORC_MAPPERJITLINKMEMORYMANAGER_H + +#include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/MemoryMapper.h" + +namespace llvm { +namespace orc { + +class MapperJITLinkMemoryManager : public jitlink::JITLinkMemoryManager { +public: + MapperJITLinkMemoryManager(std::unique_ptr Mapper); + + template + static Expected> + CreateWithMapper(Args &&...A) { + auto Mapper = MemoryMapperType::Create(std::forward(A)...); + if (!Mapper) + return Mapper.takeError(); + + return std::make_unique(std::move(*Mapper)); + } + + void allocate(const jitlink::JITLinkDylib *JD, jitlink::LinkGraph &G, + OnAllocatedFunction OnAllocated) override; + // synchronous overload + using JITLinkMemoryManager::allocate; + + void deallocate(std::vector Allocs, + OnDeallocatedFunction OnDeallocated) override; + // synchronous overload + using JITLinkMemoryManager::deallocate; + +private: + class InFlightAlloc; + + std::unique_ptr Mapper; +}; + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_MAPPERJITLINKMEMORYMANAGER_H diff --git a/llvm/include/llvm/ExecutionEngine/Orc/MemoryMapper.h b/llvm/include/llvm/ExecutionEngine/Orc/MemoryMapper.h --- a/llvm/include/llvm/ExecutionEngine/Orc/MemoryMapper.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/MemoryMapper.h @@ -14,6 +14,7 @@ #define LLVM_EXECUTIONENGINE_ORC_MEMORYMAPPER_H #include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/Support/Process.h" #include @@ -41,6 +42,9 @@ using OnReservedFunction = unique_function)>; + // Page size of the target process + virtual unsigned int getPageSize() = 0; + /// Reserves address space in executor process virtual void reserve(size_t NumBytes, OnReservedFunction OnReserved) = 0; @@ -76,7 +80,11 @@ class InProcessMemoryMapper final : public MemoryMapper { public: - InProcessMemoryMapper() {} + InProcessMemoryMapper(size_t PageSize); + + static Expected> Create(); + + unsigned int getPageSize() override { return PageSize; } void reserve(size_t NumBytes, OnReservedFunction OnReserved) override; @@ -107,6 +115,8 @@ std::mutex Mutex; ReservationMap Reservations; AllocationMap Allocations; + + size_t PageSize; }; class SharedMemoryMapper final : public MemoryMapper { @@ -119,8 +129,13 @@ ExecutorAddr Release; }; - SharedMemoryMapper(ExecutorProcessControl &EPC, SymbolAddrs SAs) - : EPC(EPC), SAs(SAs) {} + SharedMemoryMapper(ExecutorProcessControl &EPC, SymbolAddrs SAs, + size_t PageSize); + + static Expected> + Create(ExecutorProcessControl &EPC, SymbolAddrs SAs); + + unsigned int getPageSize() override { return PageSize; } void reserve(size_t NumBytes, OnReservedFunction OnReserved) override; @@ -148,6 +163,8 @@ std::mutex Mutex; std::map Reservations; + + size_t PageSize; }; } // namespace orc diff --git a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt --- a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt +++ b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt @@ -27,6 +27,7 @@ LookupAndRecordAddrs.cpp LLJIT.cpp MachOPlatform.cpp + MapperJITLinkMemoryManager.cpp MemoryMapper.cpp ELFNixPlatform.cpp Mangling.cpp diff --git a/llvm/lib/ExecutionEngine/Orc/MapperJITLinkMemoryManager.cpp b/llvm/lib/ExecutionEngine/Orc/MapperJITLinkMemoryManager.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/MapperJITLinkMemoryManager.cpp @@ -0,0 +1,135 @@ +//=== MapperJITLinkMemoryManager.cpp - Memory management with MemoryMapper ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/MapperJITLinkMemoryManager.h" + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/Support/Process.h" + +#include + +using namespace llvm::jitlink; + +namespace llvm { +namespace orc { + +class MapperJITLinkMemoryManager::InFlightAlloc + : public JITLinkMemoryManager::InFlightAlloc { +public: + InFlightAlloc(MapperJITLinkMemoryManager &Parent, LinkGraph &G, + ExecutorAddr AllocAddr, + std::vector Segs) + : Parent(Parent), G(G), AllocAddr(AllocAddr), Segs(std::move(Segs)) {} + + void finalize(OnFinalizedFunction OnFinalize) override { + MemoryMapper::AllocInfo AI; + AI.MappingBase = AllocAddr; + + std::swap(AI.Segments, Segs); + std::swap(AI.Actions, G.allocActions()); + + Parent.Mapper->initialize(AI, [&](Expected Result) { + if (!Result) { + OnFinalize(Result.takeError()); + return; + } + + OnFinalize(FinalizedAlloc(*Result)); + }); + } + + void abandon(OnAbandonedFunction OnFinalize) override { + Parent.Mapper->release({AllocAddr}, std::move(OnFinalize)); + } + +private: + MapperJITLinkMemoryManager &Parent; + LinkGraph &G; + ExecutorAddr AllocAddr; + std::vector Segs; +}; + +MapperJITLinkMemoryManager::MapperJITLinkMemoryManager( + std::unique_ptr Mapper) + : Mapper(std::move(Mapper)) {} + +void MapperJITLinkMemoryManager::allocate(const JITLinkDylib *JD, LinkGraph &G, + OnAllocatedFunction OnAllocated) { + BasicLayout BL(G); + + // find required address space + auto SegsSizes = BL.getContiguousPageBasedLayoutSizes(Mapper->getPageSize()); + if (!SegsSizes) { + OnAllocated(SegsSizes.takeError()); + return; + } + + // Check if total size fits in address space + if (SegsSizes->total() > std::numeric_limits::max()) { + OnAllocated(make_error( + formatv("Total requested size {:x} for graph {} exceeds address space", + SegsSizes->total(), G.getName()))); + return; + } + + Mapper->reserve( + SegsSizes->total(), + [this, &G, BL = std::move(BL), OnAllocated = std::move(OnAllocated)]( + Expected Result) mutable { + if (!Result) { + return OnAllocated(Result.takeError()); + } + + auto NextSegAddr = Result->Start; + + std::vector SegInfos; + + for (auto &KV : BL.segments()) { + auto &AG = KV.first; + auto &Seg = KV.second; + + auto TotalSize = Seg.ContentSize + Seg.ZeroFillSize; + + Seg.Addr = NextSegAddr; + Seg.WorkingMem = Mapper->prepare(NextSegAddr, TotalSize); + + NextSegAddr += alignTo(TotalSize, Mapper->getPageSize()); + + MemoryMapper::AllocInfo::SegInfo SI; + SI.Offset = Seg.Addr - Result->Start; + SI.ContentSize = Seg.ContentSize; + SI.ZeroFillSize = Seg.ZeroFillSize; + SI.Prot = (toSysMemoryProtectionFlags(AG.getMemProt())); + SI.WorkingMem = Seg.WorkingMem; + + SegInfos.push_back(SI); + } + + if (auto Err = BL.apply()) { + OnAllocated(std::move(Err)); + return; + } + + OnAllocated(std::make_unique(*this, G, Result->Start, + std::move(SegInfos))); + }); +} + +void MapperJITLinkMemoryManager::deallocate( + std::vector Allocs, OnDeallocatedFunction OnDeallocated) { + std::vector Bases; + Bases.reserve(Allocs.size()); + for (auto &FA : Allocs) { + Bases.push_back(FA.getAddress()); + FA.release(); + } + Mapper->release(Bases, std::move(OnDeallocated)); +} + +} // end namespace orc +} // end namespace llvm diff --git a/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp b/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp --- a/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp +++ b/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp @@ -24,6 +24,17 @@ MemoryMapper::~MemoryMapper() {} +InProcessMemoryMapper::InProcessMemoryMapper(size_t PageSize) + : PageSize(PageSize) {} + +Expected> +InProcessMemoryMapper::Create() { + auto PageSize = sys::Process::getPageSize(); + if (!PageSize) + return PageSize.takeError(); + return std::make_unique(*PageSize); +} + void InProcessMemoryMapper::reserve(size_t NumBytes, OnReservedFunction OnReserved) { std::error_code EC; @@ -160,6 +171,19 @@ // SharedMemoryMapper +SharedMemoryMapper::SharedMemoryMapper(ExecutorProcessControl &EPC, + SymbolAddrs SAs, size_t PageSize) + : EPC(EPC), SAs(SAs), PageSize(PageSize) {} + +Expected> +SharedMemoryMapper::Create(ExecutorProcessControl &EPC, SymbolAddrs SAs) { + auto PageSize = sys::Process::getPageSize(); + if (!PageSize) + return PageSize.takeError(); + + return std::make_unique(EPC, SAs, *PageSize); +} + void SharedMemoryMapper::reserve(size_t NumBytes, OnReservedFunction OnReserved) { #if defined(LLVM_ON_UNIX) || defined(_WIN32) diff --git a/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt b/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt --- a/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt +++ b/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt @@ -24,6 +24,7 @@ JITTargetMachineBuilderTest.cpp LazyCallThroughAndReexportsTest.cpp LookupAndRecordAddrsTest.cpp + MapperJITLinkMemoryManagerTest.cpp MemoryMapperTest.cpp ObjectLinkingLayerTest.cpp OrcCAPITest.cpp diff --git a/llvm/unittests/ExecutionEngine/Orc/MapperJITLinkMemoryManagerTest.cpp b/llvm/unittests/ExecutionEngine/Orc/MapperJITLinkMemoryManagerTest.cpp new file mode 100644 --- /dev/null +++ b/llvm/unittests/ExecutionEngine/Orc/MapperJITLinkMemoryManagerTest.cpp @@ -0,0 +1,48 @@ +//===-------------- MapperJITLinkMemoryManagerTest.cpp ----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "OrcTestCommon.h" + +#include "llvm/ExecutionEngine/Orc/MapperJITLinkMemoryManager.h" + +#include "llvm/Testing/Support/Error.h" + +#include + +using namespace llvm; +using namespace llvm::jitlink; +using namespace llvm::orc; +using namespace llvm::orc::shared; + +namespace { + +TEST(MapperJITLinkMemoryManagerTest, InProcess) { + auto MemMgr = cantFail( + MapperJITLinkMemoryManager::CreateWithMapper()); + + StringRef Hello = "hello"; + auto SSA = jitlink::SimpleSegmentAlloc::Create( + *MemMgr, nullptr, {{jitlink::MemProt::Read, {Hello.size(), Align(1)}}}); + EXPECT_THAT_EXPECTED(SSA, Succeeded()); + auto SegInfo = SSA->getSegInfo(jitlink::MemProt::Read); + memcpy(SegInfo.WorkingMem.data(), Hello.data(), Hello.size()); + + auto FA = SSA->finalize(); + EXPECT_THAT_EXPECTED(FA, Succeeded()); + + ExecutorAddr TargetAddr(SegInfo.Addr); + + const char *TargetMem = TargetAddr.toPtr(); + StringRef TargetHello(TargetMem, Hello.size()); + EXPECT_EQ(Hello, TargetHello); + + auto Err2 = MemMgr->deallocate(std::move(*FA)); + EXPECT_THAT_ERROR(std::move(Err2), Succeeded()); +} + +} // namespace diff --git a/llvm/unittests/ExecutionEngine/Orc/MemoryMapperTest.cpp b/llvm/unittests/ExecutionEngine/Orc/MemoryMapperTest.cpp --- a/llvm/unittests/ExecutionEngine/Orc/MemoryMapperTest.cpp +++ b/llvm/unittests/ExecutionEngine/Orc/MemoryMapperTest.cpp @@ -66,10 +66,10 @@ int DeinitializeCounter = 0; { std::unique_ptr Mapper = - std::make_unique(); + cantFail(InProcessMemoryMapper::Create()); // We will do two separate allocations - auto PageSize = cantFail(sys::Process::getPageSize()); + auto PageSize = Mapper->getPageSize(); auto TotalSize = PageSize * 2; // Reserve address space diff --git a/llvm/unittests/ExecutionEngine/Orc/SharedMemoryMapperTest.cpp b/llvm/unittests/ExecutionEngine/Orc/SharedMemoryMapperTest.cpp --- a/llvm/unittests/ExecutionEngine/Orc/SharedMemoryMapperTest.cpp +++ b/llvm/unittests/ExecutionEngine/Orc/SharedMemoryMapperTest.cpp @@ -62,11 +62,11 @@ auto F = P.get_future(); { - auto PageSize = cantFail(sys::Process::getPageSize()); - size_t ReqSize = PageSize; - std::unique_ptr Mapper = - std::make_unique(*SelfEPC, SAs); + cantFail(SharedMemoryMapper::Create(*SelfEPC, SAs)); + + auto PageSize = Mapper->getPageSize(); + size_t ReqSize = PageSize; Mapper->reserve(ReqSize, [&](Expected Result) { EXPECT_THAT_ERROR(Result.takeError(), Succeeded());