Index: include/llvm/ExecutionEngine/SectionMemoryManager.h =================================================================== --- include/llvm/ExecutionEngine/SectionMemoryManager.h +++ include/llvm/ExecutionEngine/SectionMemoryManager.h @@ -84,24 +84,17 @@ private: struct MemoryGroup { - // PendingMem contains all allocated memory blocks - // which have not yet had their permissions set. Note - // that this tracks memory blocks that have been given to - // this memory manager by the system, not those - // given out to the user. In particular, the memory manager - // will give out subblocks of these MemoryBlocks in response - // to user requests. We track which subblocks have not beeen - // given out yet in `FreeMem`. - SmallVector PendingMem; - SmallVector FreeMem; - - // All allocated memory blocks that have had their permissions - // set (i.e. that have been finalized). Because of this, we may - // not give out subblocks of this memory to the user anymore, - // even if those subblocks have not been previously given out. - SmallVector AllocatedMem; - - sys::MemoryBlock Near; + // PendingMem contains all blocks of memory (subblocks of AllocatedMem) + // which have not yet had their permissions applied, but have been given + // out to the user. FreeMem contains all block of memory, which have + // neither had their permissions applied, nor been given out to the user. + SmallVector PendingMem; + SmallVector FreeMem; + + // All memory blocks that have been requested from the system + SmallVector AllocatedMem; + + sys::MemoryBlock Near; }; uint8_t *allocateSection(MemoryGroup &MemGroup, uintptr_t Size, Index: include/llvm/Support/Memory.h =================================================================== --- include/llvm/Support/Memory.h +++ include/llvm/Support/Memory.h @@ -147,6 +147,14 @@ /// boundaries, and the JIT internal allocations are not page aligned. static bool setWritable(MemoryBlock &M, std::string *ErrMsg = nullptr); + /// trimBlockToPageSize - Return the largest block contained in M that + /// starts and ends on a page boundary + static MemoryBlock trimBlockToPageSize(MemoryBlock M); + + /// expandBlockToPageSize - Return the smallest block that contains M + /// and starts and ends on a page boundary + static MemoryBlock expandBlockToPageSize(MemoryBlock M); + /// setRangeExecutable - Mark the page containing a range of addresses /// as executable. static bool setRangeExecutable(const void *Addr, size_t Size); Index: lib/ExecutionEngine/SectionMemoryManager.cpp =================================================================== --- lib/ExecutionEngine/SectionMemoryManager.cpp +++ lib/ExecutionEngine/SectionMemoryManager.cpp @@ -54,6 +54,10 @@ uintptr_t EndOfBlock = Addr + MB.size(); // Align the address. Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1); + + // The part of the block we're giving out to the user is now pending + MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size)); + // Store cutted free memory block. MB = sys::MemoryBlock((void *)(Addr + Size), EndOfBlock - Addr - Size); return (uint8_t*)Addr; @@ -83,13 +87,17 @@ // Save this address as the basis for our next request MemGroup.Near = MB; - MemGroup.PendingMem.push_back(MB); + // Remember that we allocated this memory + MemGroup.AllocatedMem.push_back(MB); Addr = (uintptr_t)MB.base(); uintptr_t EndOfBlock = Addr + MB.size(); // Align the address. Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1); + // The part of the block we're giving out to the user is now pending + MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size)); + // The allocateMappedMemory may allocate much more memory than we need. In // this case, we store the unused memory as a free memory block. unsigned FreeSize = EndOfBlock-Addr-Size; @@ -105,9 +113,6 @@ // FIXME: Should in-progress permissions be reverted if an error occurs? std::error_code ec; - // Don't allow free memory blocks to be used after setting protection flags. - CodeMem.FreeMem.clear(); - // Make code memory executable. ec = applyMemoryGroupPermissions(CodeMem, sys::Memory::MF_READ | sys::Memory::MF_EXEC); @@ -138,25 +143,54 @@ // relocations) will get to the data cache but not to the instruction cache. invalidateInstructionCache(); - // Now, remember that we have successfully applied the permissions to avoid - // having to apply them again. - CodeMem.AllocatedMem.append(CodeMem.PendingMem.begin(),CodeMem.PendingMem.end()); - CodeMem.PendingMem.clear(); - - RODataMem.AllocatedMem.append(RODataMem.PendingMem.begin(),RODataMem.PendingMem.end()); - RODataMem.PendingMem.clear(); - return false; } std::error_code SectionMemoryManager::applyMemoryGroupPermissions(MemoryGroup &MemGroup, unsigned Permissions) { + // Sort and merge PendingMem, to mimimize the number of required system calls + std::sort(MemGroup.PendingMem.begin(), MemGroup.PendingMem.end(), + [](sys::MemoryBlock A, sys::MemoryBlock B) { + return A.base() < B.base(); + }); + + sys::MemoryBlock AccumMB(NULL, 0); + for (sys::MemoryBlock &MB : MemGroup.PendingMem) { + MB = sys::Memory::expandBlockToPageSize(MB); + if (AccumMB.base() <= MB.base() && + (uintptr_t)MB.base() <= (uintptr_t)AccumMB.base() + AccumMB.size()) + AccumMB = sys::MemoryBlock( + AccumMB.base(), + std::max(AccumMB.size(), ((uintptr_t)MB.base() + MB.size()) - + (uintptr_t)AccumMB.base())); + else { + if (AccumMB.base() != NULL) + if (std::error_code EC = + sys::Memory::protectMappedMemory(AccumMB, Permissions)) + return EC; + AccumMB = MB; + } + } - for (sys::MemoryBlock &MB : MemGroup.PendingMem) - if (std::error_code EC = sys::Memory::protectMappedMemory(MB, Permissions)) + if (AccumMB.base() != NULL) + if (std::error_code EC = + sys::Memory::protectMappedMemory(AccumMB, Permissions)) return EC; + MemGroup.PendingMem.clear(); + + // Now go through free blocks and trim any of them that don't span the entire + // page because one of the pending blocks may have overlapped it. + for (sys::MemoryBlock &MB : MemGroup.FreeMem) + MB = sys::Memory::trimBlockToPageSize(MB); + + // Remove all blocks which are now empty + MemGroup.FreeMem.erase( + std::remove_if(MemGroup.FreeMem.begin(), MemGroup.FreeMem.end(), + [](sys::MemoryBlock &MB) { return MB.size() == 0; }), + MemGroup.FreeMem.end()); + return std::error_code(); } Index: lib/Support/Memory.cpp =================================================================== --- lib/Support/Memory.cpp +++ lib/Support/Memory.cpp @@ -23,3 +23,41 @@ #ifdef LLVM_ON_WIN32 #include "Windows/Memory.inc" #endif + +namespace llvm { +namespace sys { + +MemoryBlock Memory::trimBlockToPageSize(MemoryBlock M) { + static const size_t PageSize = Process::getPageSize(); + + size_t StartOverlap = + (PageSize - ((uintptr_t)M.Address % PageSize)) % PageSize; + + MemoryBlock Trimmed((void *)((uintptr_t)M.Address + StartOverlap), M.Size); + Trimmed.Size -= StartOverlap; + Trimmed.Size -= Trimmed.Size % PageSize; + + assert(((uintptr_t)Trimmed.Address % PageSize) == 0); + assert((Trimmed.Size % PageSize) == 0); + assert(M.Address <= Trimmed.Address && Trimmed.Size <= M.Size); + + return Trimmed; +} + +MemoryBlock Memory::expandBlockToPageSize(MemoryBlock M) { + static const size_t PageSize = Process::getPageSize(); + + uintptr_t Start = (uintptr_t)M.Address - ((uintptr_t)M.Address % PageSize); + uintptr_t Stop = + PageSize * (((uintptr_t)M.Address + M.Size + PageSize - 1) / PageSize); + + MemoryBlock Expanded((void *)Start, Stop - Start); + + assert(((uintptr_t)Expanded.Address % PageSize) == 0); + assert((Expanded.Size % PageSize) == 0); + assert(Expanded.Address <= M.Address && Expanded.Size >= M.Size); + + return Expanded; +} +} +} Index: lib/Support/Unix/Memory.inc =================================================================== --- lib/Support/Unix/Memory.inc +++ lib/Support/Unix/Memory.inc @@ -152,6 +152,7 @@ std::error_code Memory::protectMappedMemory(const MemoryBlock &M, unsigned Flags) { + static const size_t PageSize = Process::getPageSize(); if (M.Address == nullptr || M.Size == 0) return std::error_code(); @@ -160,7 +161,7 @@ int Protect = getPosixProtectionFlags(Flags); - int Result = ::mprotect(M.Address, M.Size, Protect); + int Result = ::mprotect((void*)((uintptr_t)M.Address & ~(PageSize-1)), PageSize*((M.Size+PageSize-1)/PageSize), Protect); if (Result != 0) return std::error_code(errno, std::generic_category()); @@ -180,7 +181,7 @@ std::string *ErrMsg) { if (NumBytes == 0) return MemoryBlock(); - size_t PageSize = Process::getPageSize(); + static const size_t PageSize = Process::getPageSize(); size_t NumPages = (NumBytes+PageSize-1)/PageSize; int fd = -1;