diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp --- a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp +++ b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp @@ -533,250 +533,6 @@ uint64_t DeltaAddr; }; -class JITLinkSlabAllocator final : public JITLinkMemoryManager { -private: - struct FinalizedAllocInfo { - FinalizedAllocInfo(sys::MemoryBlock Mem, - std::vector DeallocActions) - : Mem(Mem), DeallocActions(std::move(DeallocActions)) {} - sys::MemoryBlock Mem; - std::vector DeallocActions; - }; - -public: - static Expected> - Create(uint64_t SlabSize) { - Error Err = Error::success(); - std::unique_ptr Allocator( - new JITLinkSlabAllocator(SlabSize, Err)); - if (Err) - return std::move(Err); - return std::move(Allocator); - } - - void allocate(const JITLinkDylib *JD, LinkGraph &G, - OnAllocatedFunction OnAllocated) override { - - // Local class for allocation. - class IPMMAlloc : public InFlightAlloc { - public: - IPMMAlloc(JITLinkSlabAllocator &Parent, BasicLayout BL, - sys::MemoryBlock StandardSegs, sys::MemoryBlock FinalizeSegs) - : Parent(Parent), BL(std::move(BL)), - StandardSegs(std::move(StandardSegs)), - FinalizeSegs(std::move(FinalizeSegs)) {} - - void finalize(OnFinalizedFunction OnFinalized) override { - if (auto Err = applyProtections()) { - OnFinalized(std::move(Err)); - return; - } - - auto DeallocActions = runFinalizeActions(BL.graphAllocActions()); - if (!DeallocActions) { - OnFinalized(DeallocActions.takeError()); - return; - } - - if (auto Err = Parent.freeBlock(FinalizeSegs)) { - OnFinalized( - joinErrors(std::move(Err), runDeallocActions(*DeallocActions))); - return; - } - - OnFinalized(FinalizedAlloc(ExecutorAddr::fromPtr( - new FinalizedAllocInfo(StandardSegs, std::move(*DeallocActions))))); - } - - void abandon(OnAbandonedFunction OnAbandoned) override { - OnAbandoned(joinErrors(Parent.freeBlock(StandardSegs), - Parent.freeBlock(FinalizeSegs))); - } - - private: - Error applyProtections() { - for (auto &KV : BL.segments()) { - const auto &Group = KV.first; - auto &Seg = KV.second; - - auto Prot = toSysMemoryProtectionFlags(Group.getMemProt()); - - uint64_t SegSize = - alignTo(Seg.ContentSize + Seg.ZeroFillSize, Parent.PageSize); - sys::MemoryBlock MB(Seg.WorkingMem, SegSize); - if (auto EC = sys::Memory::protectMappedMemory(MB, Prot)) - return errorCodeToError(EC); - if (Prot & sys::Memory::MF_EXEC) - sys::Memory::InvalidateInstructionCache(MB.base(), - MB.allocatedSize()); - } - return Error::success(); - } - - JITLinkSlabAllocator &Parent; - BasicLayout BL; - sys::MemoryBlock StandardSegs; - sys::MemoryBlock FinalizeSegs; - }; - - BasicLayout BL(G); - auto SegsSizes = BL.getContiguousPageBasedLayoutSizes(PageSize); - - if (!SegsSizes) { - OnAllocated(SegsSizes.takeError()); - return; - } - - char *AllocBase = nullptr; - { - std::lock_guard Lock(SlabMutex); - - if (SegsSizes->total() > SlabRemaining.allocatedSize()) { - OnAllocated(make_error( - "Slab allocator out of memory: request for " + - formatv("{0:x}", SegsSizes->total()) + - " bytes exceeds remaining capacity of " + - formatv("{0:x}", SlabRemaining.allocatedSize()) + " bytes", - inconvertibleErrorCode())); - return; - } - - AllocBase = reinterpret_cast(SlabRemaining.base()); - SlabRemaining = - sys::MemoryBlock(AllocBase + SegsSizes->total(), - SlabRemaining.allocatedSize() - SegsSizes->total()); - } - - sys::MemoryBlock StandardSegs(AllocBase, SegsSizes->StandardSegs); - sys::MemoryBlock FinalizeSegs(AllocBase + SegsSizes->StandardSegs, - SegsSizes->FinalizeSegs); - - auto NextStandardSegAddr = ExecutorAddr::fromPtr(StandardSegs.base()); - auto NextFinalizeSegAddr = ExecutorAddr::fromPtr(FinalizeSegs.base()); - - LLVM_DEBUG({ - dbgs() << "JITLinkSlabAllocator allocated:\n"; - if (SegsSizes->StandardSegs) - dbgs() << formatv(" [ {0:x16} -- {1:x16} ]", NextStandardSegAddr, - NextStandardSegAddr + StandardSegs.allocatedSize()) - << " to stardard segs\n"; - else - dbgs() << " no standard segs\n"; - if (SegsSizes->FinalizeSegs) - dbgs() << formatv(" [ {0:x16} -- {1:x16} ]", NextFinalizeSegAddr, - NextFinalizeSegAddr + FinalizeSegs.allocatedSize()) - << " to finalize segs\n"; - else - dbgs() << " no finalize segs\n"; - }); - - for (auto &KV : BL.segments()) { - auto &Group = KV.first; - auto &Seg = KV.second; - - auto &SegAddr = - (Group.getMemDeallocPolicy() == MemDeallocPolicy::Standard) - ? NextStandardSegAddr - : NextFinalizeSegAddr; - - LLVM_DEBUG({ - dbgs() << " " << Group << " -> " << formatv("{0:x16}", SegAddr) - << "\n"; - }); - Seg.WorkingMem = SegAddr.toPtr(); - Seg.Addr = SegAddr + SlabDelta; - - SegAddr += alignTo(Seg.ContentSize + Seg.ZeroFillSize, PageSize); - - // Zero out the zero-fill memory. - if (Seg.ZeroFillSize != 0) - memset(Seg.WorkingMem + Seg.ContentSize, 0, Seg.ZeroFillSize); - } - - if (auto Err = BL.apply()) { - OnAllocated(std::move(Err)); - return; - } - - OnAllocated(std::unique_ptr( - new IPMMAlloc(*this, std::move(BL), std::move(StandardSegs), - std::move(FinalizeSegs)))); - } - - void deallocate(std::vector FinalizedAllocs, - OnDeallocatedFunction OnDeallocated) override { - Error Err = Error::success(); - for (auto &FA : FinalizedAllocs) { - std::unique_ptr FAI( - FA.release().toPtr()); - - // FIXME: Run dealloc actions. - - Err = joinErrors(std::move(Err), freeBlock(FAI->Mem)); - } - OnDeallocated(std::move(Err)); - } - -private: - JITLinkSlabAllocator(uint64_t SlabSize, Error &Err) { - ErrorAsOutParameter _(&Err); - - if (!SlabPageSize) { - if (auto PageSizeOrErr = sys::Process::getPageSize()) - PageSize = *PageSizeOrErr; - else { - Err = PageSizeOrErr.takeError(); - return; - } - - if (PageSize == 0) { - Err = make_error("Page size is zero", - inconvertibleErrorCode()); - return; - } - } else - PageSize = SlabPageSize; - - if (!isPowerOf2_64(PageSize)) { - Err = make_error("Page size is not a power of 2", - inconvertibleErrorCode()); - return; - } - - // Round slab request up to page size. - SlabSize = (SlabSize + PageSize - 1) & ~(PageSize - 1); - - const sys::Memory::ProtectionFlags ReadWrite = - static_cast(sys::Memory::MF_READ | - sys::Memory::MF_WRITE); - - std::error_code EC; - SlabRemaining = - sys::Memory::allocateMappedMemory(SlabSize, nullptr, ReadWrite, EC); - - if (EC) { - Err = errorCodeToError(EC); - return; - } - - // Calculate the target address delta to link as-if slab were at - // SlabAddress. - if (SlabAddress != ~0ULL) - SlabDelta = ExecutorAddr(SlabAddress) - - ExecutorAddr::fromPtr(SlabRemaining.base()); - } - - Error freeBlock(sys::MemoryBlock MB) { - // FIXME: Return memory to slab. - return Error::success(); - } - - std::mutex SlabMutex; - sys::MemoryBlock SlabRemaining; - uint64_t PageSize = 0; - int64_t SlabDelta = 0; -}; - Expected getSlabAllocSize(StringRef SizeString) { SizeString = SizeString.trim();