diff --git a/llvm/include/llvm/Support/Alignment.h b/llvm/include/llvm/Support/Alignment.h --- a/llvm/include/llvm/Support/Alignment.h +++ b/llvm/include/llvm/Support/Alignment.h @@ -154,6 +154,11 @@ return SizeInBytes % (*Lhs).value() == 0; } +/// Checks that Addr is a multiple of the alignment. +inline bool isAddrAligned(Align Lhs, const void *Addr) { + return isAligned(Lhs, reinterpret_cast(Addr)); +} + /// Returns a multiple of A needed to store `Size` bytes. inline uint64_t alignTo(uint64_t Size, Align A) { return (Size + A.value() - 1) / A.value() * A.value(); @@ -165,12 +170,25 @@ return A ? alignTo(Size, A.getValue()) : Size; } +/// Aligns `Addr` to `Alignment` bytes, rounding up. +inline uintptr_t alignAddr(const void *Addr, Align Alignment) { + uintptr_t ArithAddr = reinterpret_cast(Addr); + assert(ArithAddr + Alignment.value() - 1 >= ArithAddr && "Overflow"); + return alignTo(ArithAddr, Alignment); +} + /// Returns the offset to the next integer (mod 2**64) that is greater than /// or equal to \p Value and is a multiple of \p Align. inline uint64_t offsetToAlignment(uint64_t Value, Align Alignment) { return alignTo(Value, Alignment) - Value; } +/// Returns the necessary adjustment for aligning `Addr` to `Alignment` +/// bytes, rounding up. +inline uint64_t offsetToAlignedAddr(const void *Addr, Align Alignment) { + return offsetToAlignment(reinterpret_cast(Addr), Alignment); +} + /// Returns the log2 of the alignment. inline unsigned Log2(Align A) { return A.ShiftValue; } diff --git a/llvm/include/llvm/Support/Allocator.h b/llvm/include/llvm/Support/Allocator.h --- a/llvm/include/llvm/Support/Allocator.h +++ b/llvm/include/llvm/Support/Allocator.h @@ -22,6 +22,7 @@ #include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Alignment.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" @@ -211,13 +212,11 @@ /// Allocate space at the specified alignment. LLVM_ATTRIBUTE_RETURNS_NONNULL LLVM_ATTRIBUTE_RETURNS_NOALIAS void * - Allocate(size_t Size, size_t Alignment) { - assert(Alignment > 0 && "0-byte alignnment is not allowed. Use 1 instead."); - + Allocate(size_t Size, Align Alignment) { // Keep track of how many bytes we've allocated. BytesAllocated += Size; - size_t Adjustment = alignmentAdjustment(CurPtr, Alignment); + size_t Adjustment = offsetToAlignedAddr(CurPtr, Alignment); assert(Adjustment + Size >= Size && "Adjustment + Size must not overflow"); size_t SizeToAllocate = Size; @@ -240,7 +239,7 @@ } // If Size is really big, allocate a separate slab for it. - size_t PaddedSize = SizeToAllocate + Alignment - 1; + size_t PaddedSize = SizeToAllocate + Alignment.value() - 1; if (PaddedSize > SizeThreshold) { void *NewSlab = Allocator.Allocate(PaddedSize, 0); // We own the new slab and don't want anyone reading anyting other than @@ -268,6 +267,12 @@ return AlignedPtr; } + inline LLVM_ATTRIBUTE_RETURNS_NONNULL LLVM_ATTRIBUTE_RETURNS_NOALIAS void * + Allocate(size_t Size, size_t Alignment) { + assert(Alignment > 0 && "0-byte alignnment is not allowed. Use 1 instead."); + return Allocate(Size, Align(Alignment)); + } + // Pull in base class overloads. using AllocatorBase::Allocate; @@ -461,7 +466,7 @@ /// all memory allocated so far. void DestroyAll() { auto DestroyElements = [](char *Begin, char *End) { - assert(Begin == (char *)alignAddr(Begin, alignof(T))); + assert(Begin == (char *)alignAddr(Begin, Align::Of())); for (char *Ptr = Begin; Ptr + sizeof(T) <= End; Ptr += sizeof(T)) reinterpret_cast(Ptr)->~T(); }; @@ -470,7 +475,7 @@ ++I) { size_t AllocatedSlabSize = BumpPtrAllocator::computeSlabSize( std::distance(Allocator.Slabs.begin(), I)); - char *Begin = (char *)alignAddr(*I, alignof(T)); + char *Begin = (char *)alignAddr(*I, Align::Of()); char *End = *I == Allocator.Slabs.back() ? Allocator.CurPtr : (char *)*I + AllocatedSlabSize; @@ -480,7 +485,8 @@ for (auto &PtrAndSize : Allocator.CustomSizedSlabs) { void *Ptr = PtrAndSize.first; size_t Size = PtrAndSize.second; - DestroyElements((char *)alignAddr(Ptr, alignof(T)), (char *)Ptr + Size); + DestroyElements((char *)alignAddr(Ptr, Align::Of()), + (char *)Ptr + Size); } Allocator.Reset(); diff --git a/llvm/include/llvm/Support/BinaryStreamArray.h b/llvm/include/llvm/Support/BinaryStreamArray.h --- a/llvm/include/llvm/Support/BinaryStreamArray.h +++ b/llvm/include/llvm/Support/BinaryStreamArray.h @@ -286,7 +286,7 @@ // an exact multiple of the element size. consumeError(std::move(EC)); } - assert(llvm::alignmentAdjustment(Data.data(), alignof(T)) == 0); + assert(isAddrAligned(Align::Of(), Data.data())); return *reinterpret_cast(Data.data()); } diff --git a/llvm/include/llvm/Support/BinaryStreamReader.h b/llvm/include/llvm/Support/BinaryStreamReader.h --- a/llvm/include/llvm/Support/BinaryStreamReader.h +++ b/llvm/include/llvm/Support/BinaryStreamReader.h @@ -198,7 +198,7 @@ if (auto EC = readBytes(Bytes, NumElements * sizeof(T))) return EC; - assert(alignmentAdjustment(Bytes.data(), alignof(T)) == 0 && + assert(isAddrAligned(Align::Of(), Bytes.data()) && "Reading at invalid alignment!"); Array = ArrayRef(reinterpret_cast(Bytes.data()), NumElements); diff --git a/llvm/include/llvm/Support/MathExtras.h b/llvm/include/llvm/Support/MathExtras.h --- a/llvm/include/llvm/Support/MathExtras.h +++ b/llvm/include/llvm/Support/MathExtras.h @@ -667,25 +667,6 @@ return (A | B) & (1 + ~(A | B)); } -/// Aligns \c Addr to \c Alignment bytes, rounding up. -/// -/// Alignment should be a power of two. This method rounds up, so -/// alignAddr(7, 4) == 8 and alignAddr(8, 4) == 8. -inline uintptr_t alignAddr(const void *Addr, size_t Alignment) { - assert(Alignment && isPowerOf2_64((uint64_t)Alignment) && - "Alignment is not a power of two!"); - - assert((uintptr_t)Addr + Alignment - 1 >= (uintptr_t)Addr); - - return (((uintptr_t)Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1)); -} - -/// Returns the necessary adjustment for aligning \c Ptr to \c Alignment -/// bytes, rounding up. -inline size_t alignmentAdjustment(const void *Ptr, size_t Alignment) { - return alignAddr(Ptr, Alignment) - (uintptr_t)Ptr; -} - /// Returns the next power of two (in 64-bits) that is strictly greater than A. /// Returns zero on overflow. inline uint64_t NextPowerOf2(uint64_t A) { diff --git a/llvm/include/llvm/Support/TrailingObjects.h b/llvm/include/llvm/Support/TrailingObjects.h --- a/llvm/include/llvm/Support/TrailingObjects.h +++ b/llvm/include/llvm/Support/TrailingObjects.h @@ -47,6 +47,7 @@ #define LLVM_SUPPORT_TRAILINGOBJECTS_H #include "llvm/Support/AlignOf.h" +#include "llvm/Support/Alignment.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/type_traits.h" @@ -167,7 +168,7 @@ if (requiresRealignment()) return reinterpret_cast( - llvm::alignAddr(Ptr, alignof(NextTy))); + alignAddr(Ptr, Align::Of())); else return reinterpret_cast(Ptr); } @@ -181,7 +182,7 @@ Obj, TrailingObjectsBase::OverloadToken()); if (requiresRealignment()) - return reinterpret_cast(llvm::alignAddr(Ptr, alignof(NextTy))); + return reinterpret_cast(alignAddr(Ptr, Align::Of())); else return reinterpret_cast(Ptr); } diff --git a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp --- a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp +++ b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp @@ -506,7 +506,7 @@ return make_error(coveragemap_error::malformed); // Each coverage map has an alignment of 8, so we need to adjust alignment // before reading the next map. - Buf += alignmentAdjustment(Buf, 8); + Buf += offsetToAlignedAddr(Buf, Align(8)); auto CFR = reinterpret_cast(FunBuf); while ((const char *)CFR < FunEnd) { @@ -648,7 +648,7 @@ // Skip the padding bytes because coverage map data has an alignment of 8. if (CoverageMapping.empty()) return make_error(coveragemap_error::truncated); - size_t Pad = alignmentAdjustment(CoverageMapping.data(), 8); + size_t Pad = offsetToAlignedAddr(CoverageMapping.data(), Align(8)); if (CoverageMapping.size() < Pad) return make_error(coveragemap_error::malformed); CoverageMapping = CoverageMapping.substr(Pad); diff --git a/llvm/lib/Support/Unix/Memory.inc b/llvm/lib/Support/Unix/Memory.inc --- a/llvm/lib/Support/Unix/Memory.inc +++ b/llvm/lib/Support/Unix/Memory.inc @@ -176,7 +176,7 @@ std::error_code Memory::protectMappedMemory(const MemoryBlock &M, unsigned Flags) { - static const size_t PageSize = Process::getPageSizeEstimate(); + static const Align PageSize = Align(Process::getPageSizeEstimate()); if (M.Address == nullptr || M.AllocatedSize == 0) return std::error_code(); @@ -184,8 +184,8 @@ return std::error_code(EINVAL, std::generic_category()); int Protect = getPosixProtectionFlags(Flags); - uintptr_t Start = alignAddr((uint8_t *)M.Address - PageSize + 1, PageSize); - uintptr_t End = alignAddr((uint8_t *)M.Address + M.AllocatedSize, PageSize); + uintptr_t Start = alignAddr((const uint8_t *)M.Address - PageSize.value() + 1, PageSize); + uintptr_t End = alignAddr((const uint8_t *)M.Address + M.AllocatedSize, PageSize); bool InvalidateCache = (Flags & MF_EXEC); diff --git a/llvm/unittests/Support/AlignmentTest.cpp b/llvm/unittests/Support/AlignmentTest.cpp --- a/llvm/unittests/Support/AlignmentTest.cpp +++ b/llvm/unittests/Support/AlignmentTest.cpp @@ -90,6 +90,11 @@ uint64_t alignment; uint64_t offset; uint64_t rounded; + const void *forgedAddr() const { + // A value of any integral or enumeration type can be converted to a + // pointer type. + return reinterpret_cast(offset); + } } kTests[] = { // MaybeAlign {0, 0, 0}, @@ -116,6 +121,7 @@ // Test Align if (A) { EXPECT_EQ(alignTo(T.offset, A.getValue()), T.rounded); + EXPECT_EQ(alignAddr(T.forgedAddr(), A.getValue()), T.rounded); } } } @@ -174,13 +180,17 @@ EXPECT_EQ(Expected, Actual); } -TEST(AlignmentTest, isAligned) { +TEST(AlignmentTest, isAligned_isAddrAligned) { struct { uint64_t alignment; uint64_t offset; bool isAligned; + const void *forgedAddr() const { + // A value of any integral or enumeration type can be converted to a + // pointer type. + return reinterpret_cast(offset); + } } kTests[] = { - // MaybeAlign / Align {1, 0, true}, {1, 1, true}, {1, 5, true}, {2, 0, true}, {2, 1, false}, {2, 2, true}, {2, 7, false}, {2, 16, true}, {4, 0, true}, {4, 1, false}, {4, 4, true}, {4, 6, false}, @@ -192,10 +202,32 @@ // Test Align if (A) { EXPECT_EQ(isAligned(A.getValue(), T.offset), T.isAligned); + EXPECT_EQ(isAddrAligned(A.getValue(), T.forgedAddr()), T.isAligned); } } } +TEST(AlignmentTest, offsetToAlignment) { + struct { + uint64_t alignment; + uint64_t offset; + uint64_t alignedOffset; + const void *forgedAddr() const { + // A value of any integral or enumeration type can be converted to a + // pointer type. + return reinterpret_cast(offset); + } + } kTests[] = { + {1, 0, 0}, {1, 1, 0}, {1, 5, 0}, {2, 0, 0}, {2, 1, 1}, {2, 2, 0}, + {2, 7, 1}, {2, 16, 0}, {4, 0, 0}, {4, 1, 3}, {4, 4, 0}, {4, 6, 2}, + }; + for (const auto &T : kTests) { + const Align A(T.alignment); + EXPECT_EQ(offsetToAlignment(T.offset, A), T.alignedOffset); + EXPECT_EQ(offsetToAlignedAddr(T.forgedAddr(), A), T.alignedOffset); + } +} + TEST(AlignmentTest, AlignComparisons) { std::vector ValidAlignments = getValidAlignments(); std::sort(ValidAlignments.begin(), ValidAlignments.end()); @@ -349,6 +381,12 @@ } } +TEST(AlignmentDeathTest, AlignAddr) { + const void *const unaligned_high_ptr = + reinterpret_cast(std::numeric_limits::max() - 1); + EXPECT_DEATH(alignAddr(unaligned_high_ptr, Align(16)), "Overflow"); +} + #endif // NDEBUG } // end anonymous namespace diff --git a/llvm/unittests/Support/AllocatorTest.cpp b/llvm/unittests/Support/AllocatorTest.cpp --- a/llvm/unittests/Support/AllocatorTest.cpp +++ b/llvm/unittests/Support/AllocatorTest.cpp @@ -145,8 +145,8 @@ void *Allocate(size_t Size, size_t /*Alignment*/) { // Allocate space for the alignment, the slab, and a void* that goes right // before the slab. - size_t Alignment = 4096; - void *MemBase = safe_malloc(Size + Alignment - 1 + sizeof(void*)); + Align Alignment(4096); + void *MemBase = safe_malloc(Size + Alignment.value() - 1 + sizeof(void *)); // Find the slab start. void *Slab = (void *)alignAddr((char*)MemBase + sizeof(void *), Alignment); diff --git a/llvm/unittests/Support/TrailingObjectsTest.cpp b/llvm/unittests/Support/TrailingObjectsTest.cpp --- a/llvm/unittests/Support/TrailingObjectsTest.cpp +++ b/llvm/unittests/Support/TrailingObjectsTest.cpp @@ -232,7 +232,7 @@ EXPECT_EQ(C->getTrailingObjects(), reinterpret_cast(C + 1)); EXPECT_EQ(C->getTrailingObjects(), reinterpret_cast(llvm::alignAddr( - reinterpret_cast(C + 1) + 1, alignof(long)))); + reinterpret_cast(C + 1) + 1, Align::Of()))); } }