Index: compiler-rt/trunk/lib/xray/xray_allocator.h =================================================================== --- compiler-rt/trunk/lib/xray/xray_allocator.h +++ compiler-rt/trunk/lib/xray/xray_allocator.h @@ -32,13 +32,15 @@ // internal allocator. This allows us to manage the memory directly, using // mmap'ed memory to back the allocators. template T *allocate() XRAY_NEVER_INSTRUMENT { - auto B = reinterpret_cast( - internal_mmap(NULL, sizeof(T), PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); - if (B == MAP_FAILED) { + uptr RoundedSize = RoundUpTo(sizeof(T), GetPageSizeCached()); + uptr B = internal_mmap(NULL, RoundedSize, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + int ErrNo; + if (UNLIKELY(internal_iserror(B, &ErrNo))) { if (Verbosity()) - Report("XRay Profiling: Failed to allocate memory of size %d.\n", - sizeof(T)); + Report( + "XRay Profiling: Failed to allocate memory of size %d; Error = %d.\n", + RoundedSize, B); return nullptr; } return reinterpret_cast(B); @@ -47,16 +49,20 @@ template void deallocate(T *B) XRAY_NEVER_INSTRUMENT { if (B == nullptr) return; - internal_munmap(B, sizeof(T)); + uptr RoundedSize = RoundUpTo(sizeof(T), GetPageSizeCached()); + internal_munmap(B, RoundedSize); } template T *allocateBuffer(size_t S) XRAY_NEVER_INSTRUMENT { - auto B = reinterpret_cast( - internal_mmap(NULL, S * sizeof(T), PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); - if (B == MAP_FAILED) { + uptr RoundedSize = RoundUpTo(S * sizeof(T), GetPageSizeCached()); + uptr B = internal_mmap(NULL, RoundedSize, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + int ErrNo; + if (UNLIKELY(internal_iserror(B, &ErrNo))) { if (Verbosity()) - Report("XRay Profiling: Failed to allocate memory of size %d.\n", S); + Report( + "XRay Profiling: Failed to allocate memory of size %d; Error = %d.\n", + RoundedSize, B); return nullptr; } return reinterpret_cast(B); @@ -65,7 +71,8 @@ template void deallocateBuffer(T *B, size_t S) XRAY_NEVER_INSTRUMENT { if (B == nullptr) return; - internal_munmap(B, S); + uptr RoundedSize = RoundUpTo(S * sizeof(T), GetPageSizeCached()); + internal_munmap(B, RoundedSize); } template @@ -104,19 +111,16 @@ private: const size_t MaxMemory{0}; - void *BackingStore = nullptr; - void *AlignedNextBlock = nullptr; + uint8_t *BackingStore = nullptr; + uint8_t *AlignedNextBlock = nullptr; size_t AllocatedBlocks = 0; SpinMutex Mutex{}; void *Alloc() XRAY_NEVER_INSTRUMENT { SpinMutexLock Lock(&Mutex); if (UNLIKELY(BackingStore == nullptr)) { - BackingStore = reinterpret_cast( - internal_mmap(NULL, MaxMemory, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); - if (BackingStore == MAP_FAILED) { - BackingStore = nullptr; + BackingStore = allocateBuffer(MaxMemory); + if (BackingStore == nullptr) { if (Verbosity()) Report("XRay Profiling: Failed to allocate memory for allocator.\n"); return nullptr; @@ -129,7 +133,7 @@ auto AlignedNextBlockNum = nearest_boundary( reinterpret_cast(AlignedNextBlock), kCacheLineSize); if (diff(AlignedNextBlockNum, BackingStoreNum) > ptrdiff_t(MaxMemory)) { - munmap(BackingStore, MaxMemory); + deallocateBuffer(BackingStore, MaxMemory); AlignedNextBlock = BackingStore = nullptr; if (Verbosity()) Report("XRay Profiling: Cannot obtain enough memory from " @@ -137,7 +141,7 @@ return nullptr; } - AlignedNextBlock = reinterpret_cast(AlignedNextBlockNum); + AlignedNextBlock = reinterpret_cast(AlignedNextBlockNum); // Assert that AlignedNextBlock is cache-line aligned. DCHECK_EQ(reinterpret_cast(AlignedNextBlock) % kCacheLineSize, @@ -150,8 +154,8 @@ // Align the pointer we'd like to return to an appropriate alignment, then // advance the pointer from where to start allocations. void *Result = AlignedNextBlock; - AlignedNextBlock = reinterpret_cast( - reinterpret_cast(AlignedNextBlock) + N); + AlignedNextBlock = reinterpret_cast( + reinterpret_cast(AlignedNextBlock) + N); ++AllocatedBlocks; return Result; } @@ -164,7 +168,7 @@ ~Allocator() NOEXCEPT XRAY_NEVER_INSTRUMENT { if (BackingStore != nullptr) { - internal_munmap(BackingStore, MaxMemory); + deallocateBuffer(BackingStore, MaxMemory); } } }; Index: compiler-rt/trunk/lib/xray/xray_buffer_queue.h =================================================================== --- compiler-rt/trunk/lib/xray/xray_buffer_queue.h +++ compiler-rt/trunk/lib/xray/xray_buffer_queue.h @@ -20,6 +20,7 @@ #include "sanitizer_common/sanitizer_mutex.h" #include "xray_defs.h" #include +#include namespace __xray { @@ -114,7 +115,7 @@ // A pointer to a contiguous block of memory to serve as the backing store for // all the individual buffers handed out. - void *BackingStore; + uint8_t *BackingStore; // A dynamically allocated array of BufferRep instances. BufferRep *Buffers; Index: compiler-rt/trunk/lib/xray/xray_fdr_logging.cc =================================================================== --- compiler-rt/trunk/lib/xray/xray_fdr_logging.cc +++ compiler-rt/trunk/lib/xray/xray_fdr_logging.cc @@ -721,7 +721,7 @@ static BufferQueue::const_iterator It{}; static BufferQueue::const_iterator End{}; - static void *CurrentBuffer{nullptr}; + static uint8_t *CurrentBuffer{nullptr}; static size_t SerializedBufferSize = 0; if (B.Data == static_cast(&Header) && B.Size == sizeof(Header)) { // From this point on, we provide raw access to the raw buffer we're getting Index: compiler-rt/trunk/lib/xray/xray_profile_collector.cc =================================================================== --- compiler-rt/trunk/lib/xray/xray_profile_collector.cc +++ compiler-rt/trunk/lib/xray/xray_profile_collector.cc @@ -162,34 +162,34 @@ static void serializeRecords(ProfileBuffer *Buffer, const BlockHeader &Header, const ProfileRecordArray &ProfileRecords) XRAY_NEVER_INSTRUMENT { - auto NextPtr = static_cast( + auto NextPtr = static_cast( internal_memcpy(Buffer->Data, &Header, sizeof(Header))) + sizeof(Header); for (const auto &Record : ProfileRecords) { // List of IDs follow: for (const auto FId : Record.Path) NextPtr = - static_cast(internal_memcpy(NextPtr, &FId, sizeof(FId))) + + static_cast(internal_memcpy(NextPtr, &FId, sizeof(FId))) + sizeof(FId); // Add the sentinel here. constexpr int32_t SentinelFId = 0; - NextPtr = static_cast( + NextPtr = static_cast( internal_memset(NextPtr, SentinelFId, sizeof(SentinelFId))) + sizeof(SentinelFId); // Add the node data here. NextPtr = - static_cast(internal_memcpy(NextPtr, &Record.Node->CallCount, - sizeof(Record.Node->CallCount))) + + static_cast(internal_memcpy( + NextPtr, &Record.Node->CallCount, sizeof(Record.Node->CallCount))) + sizeof(Record.Node->CallCount); - NextPtr = static_cast( + NextPtr = static_cast( internal_memcpy(NextPtr, &Record.Node->CumulativeLocalTime, sizeof(Record.Node->CumulativeLocalTime))) + sizeof(Record.Node->CumulativeLocalTime); } - DCHECK_EQ(NextPtr - static_cast(Buffer->Data), Buffer->Size); + DCHECK_EQ(NextPtr - static_cast(Buffer->Data), Buffer->Size); } } // namespace @@ -203,7 +203,7 @@ // Clear out the global ProfileBuffers, if it's not empty. for (auto &B : *ProfileBuffers) - deallocateBuffer(B.Data, B.Size); + deallocateBuffer(reinterpret_cast(B.Data), B.Size); ProfileBuffers->trim(ProfileBuffers->size()); if (ThreadTries->empty()) @@ -259,7 +259,7 @@ if (ProfileBuffers != nullptr) { // Clear out the profile buffers that have been serialized. for (auto &B : *ProfileBuffers) - deallocateBuffer(B.Data, B.Size); + deallocateBuffer(reinterpret_cast(B.Data), B.Size); ProfileBuffers->trim(ProfileBuffers->size()); }