diff --git a/llvm/include/llvm/Support/Memory.h b/llvm/include/llvm/Support/Memory.h --- a/llvm/include/llvm/Support/Memory.h +++ b/llvm/include/llvm/Support/Memory.h @@ -32,15 +32,16 @@ class MemoryBlock { public: MemoryBlock() : Address(nullptr), AllocatedSize(0) {} - MemoryBlock(void *addr, size_t allocatedSize) - : Address(addr), AllocatedSize(allocatedSize) {} + MemoryBlock(void *addr, size_t allocatedSize, unsigned Flags = 0) + : Address(addr), AllocatedSize(allocatedSize), Flags(Flags) {} void *base() const { return Address; } /// The size as it was allocated. This is always greater or equal to the /// size that was originally requested. size_t allocatedSize() const { return AllocatedSize; } - + unsigned flags() const { return Flags; } + private: - void *Address; ///< Address of first byte of memory area + void *Address; ///< Address of first byte of memory area size_t AllocatedSize; ///< Size, in bytes of the memory area unsigned Flags = 0; friend class Memory; diff --git a/llvm/lib/Support/Unix/Memory.inc b/llvm/lib/Support/Unix/Memory.inc --- a/llvm/lib/Support/Unix/Memory.inc +++ b/llvm/lib/Support/Unix/Memory.inc @@ -76,16 +76,38 @@ return PROT_NONE; } +int getHugeMmapFlags(unsigned Flags) { + if (Flags & llvm::sys::Memory::MF_HUGE_HINT) { +#if defined(MAP_HUGETLB) + return MAP_HUGETLB; // Linux >=2.6.32 +#elif defined(MAP_ALIGNED_SUPER) + return MAP_ALIGNED_SUPER; // FreeBSD >=9 +#endif + // fallthrough + } + return 0; +} + +int getHugeMadviseFlags(unsigned Flags) { + if (Flags & llvm::sys::Memory::MF_HUGE_HINT) { +#if defined(MADV_HUGEPAGE) // Linux >= 2.6.38 + return MADV_HUGEPAGE; +#endif + // fallthrough + } + return 0; +} + } // anonymous namespace namespace llvm { namespace sys { -MemoryBlock -Memory::allocateMappedMemory(size_t NumBytes, - const MemoryBlock *const NearBlock, - unsigned PFlags, - std::error_code &EC) { +static MemoryBlock allocateMappedMemoryImpl(size_t NumBytes, + const MemoryBlock *const NearBlock, + unsigned PFlags, + std::error_code &EC, + bool MmapHugePageFailed) { EC = std::error_code(); if (NumBytes == 0) return MemoryBlock(); @@ -123,36 +145,58 @@ if (Start && Start % PageSize) Start += PageSize - Start % PageSize; - // FIXME: Handle huge page requests (MF_HUGE_HINT). - void *Addr = ::mmap(reinterpret_cast(Start), PageSize*NumPages, Protect, - MMFlags, fd, 0); - if (Addr == MAP_FAILED) { - if (NearBlock) { //Try again without a near hint + // Handle huge page requests (MF_HUGE_HINT) as follow: + // 1. Try to directly request huge pages by passing the appropriate flag + // to mmap. + // 2. If the above fails, try again without the huge page request. In this + // case if supported, call madvise with a huge page hint. + // + // If we have both an address hint an a huge page hint, on failure first try + // without the huge page hint and then try without both. + int HugeMmapFlags = getHugeMmapFlags(PFlags); + + void *Addr = ::mmap(reinterpret_cast(Start), PageSize * NumPages, + Protect, MMFlags | HugeMmapFlags, fd, 0); + #if !defined(MAP_ANON) - close(fd); + close(fd); // No longer needed. #endif - return allocateMappedMemory(NumBytes, nullptr, PFlags, EC); - } + if (Addr == MAP_FAILED) { + // The failure could be due to either a huge page request, or + // an address hint. Try first without the huge page request. + if (PFlags & llvm::sys::Memory::MF_HUGE_HINT) + return allocateMappedMemoryImpl(NumBytes, NearBlock, + PFlags & ~llvm::sys::Memory::MF_HUGE_HINT, + EC, /*MmapHugePageFailed=*/true); + + // Then try again without a near hint. + if (NearBlock) + return allocateMappedMemoryImpl(NumBytes, /*NearBlock=*/nullptr, PFlags, + EC, MmapHugePageFailed); + + // Ok, we really failed. Give up. EC = std::error_code(errno, std::generic_category()); -#if !defined(MAP_ANON) - close(fd); -#endif return MemoryBlock(); } -#if !defined(MAP_ANON) - close(fd); -#endif + // Hint that we want huge pages if supported and we failed to directly ask + // for it above. If the madvise call succeeds we will set the MF_HUGE_HINT + // flag in the returned memory block. + int HugeMadviseFlags = getHugeMadviseFlags(llvm::sys::Memory::MF_HUGE_HINT); + bool HugeMadviseSuccess = false; + if (MmapHugePageFailed && HugeMadviseFlags) { + HugeMadviseSuccess = + ::madvise(Addr, PageSize * NumPages, HugeMadviseFlags) == 0; + } - MemoryBlock Result; - Result.Address = Addr; - Result.AllocatedSize = PageSize*NumPages; - Result.Flags = PFlags; + MemoryBlock Result( + Addr, PageSize * NumPages, + PFlags | (HugeMadviseSuccess ? llvm::sys::Memory::MF_HUGE_HINT : 0)); // Rely on protectMappedMemory to invalidate instruction cache. - if (PFlags & MF_EXEC) { - EC = Memory::protectMappedMemory (Result, PFlags); + if (PFlags & llvm::sys::Memory::MF_EXEC) { + EC = Memory::protectMappedMemory(Result, PFlags); if (EC != std::error_code()) return MemoryBlock(); } @@ -160,8 +204,14 @@ return Result; } -std::error_code -Memory::releaseMappedMemory(MemoryBlock &M) { +MemoryBlock Memory::allocateMappedMemory(size_t NumBytes, + const MemoryBlock *const NearBlock, + unsigned PFlags, std::error_code &EC) { + return allocateMappedMemoryImpl(NumBytes, NearBlock, PFlags, EC, + /*MmapHugePageFailed=*/false); +} + +std::error_code Memory::releaseMappedMemory(MemoryBlock &M) { if (M.Address == nullptr || M.AllocatedSize == 0) return std::error_code(); diff --git a/llvm/unittests/Support/MemoryTest.cpp b/llvm/unittests/Support/MemoryTest.cpp --- a/llvm/unittests/Support/MemoryTest.cpp +++ b/llvm/unittests/Support/MemoryTest.cpp @@ -102,6 +102,10 @@ EXPECT_NE((void*)nullptr, M1.base()); EXPECT_LE(sizeof(int), M1.allocatedSize()); + // The returned block of memory should not magically + // get the MF_HUGE_HINT flag if we did not ask for it. + EXPECT_FALSE(M1.flags() & Memory::MF_HUGE_HINT); + EXPECT_FALSE(Memory::releaseMappedMemory(M1)); }