[XRay][compiler-rt] Update use of internal_mmap

[dberris]

Summary

The implementation of internal_mmap(...) deviates from the contract of
mmap(...) -- i.e. error returns are actually the equivalent of errno
results. We update how XRay uses internal_mmap(...) to better handle
these error conditions.

In the process, we change the default pointers we're using from char*
to uint8_t* to prevent potential usage of the pointers in the string
library functions that expect to operate on char*.

We also take the chance to "promote" sizes of individual internal_mmap
requests to at least page size bytes, consistent with the expectations
of calls to mmap.

[dberris]

Update error reporting in one path.

[cryptoad]

compiler-rt/lib/xray/xray_allocator.h

125 ↗

(On Diff #166486)

There appears to be another use of internal_mmap here with a MAP_FAILED check.

[dberris]

Remove another check for MAP_FAILED.

compiler-rt/lib/xray/xray_allocator.h

125 ↗

(On Diff #166486)

Good catch, thanks!

compiler-rt/trunk/lib/xray/xray_allocator.h

 #include <sys/mman.h>
 
 namespace __xray {
 
 // We implement our own memory allocation routine which will bypass the
 // internal allocator. This allows us to manage the memory directly, using
 // mmap'ed memory to back the allocators.
 template <class T> T *allocate() XRAY_NEVER_INSTRUMENT {
-  auto B = reinterpret_cast<void *>(
-      internal_mmap(NULL, sizeof(T), PROT_READ | PROT_WRITE,
-                    MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
-  if (B == MAP_FAILED) {
+  uptr RoundedSize = RoundUpTo(sizeof(T), GetPageSizeCached());
+  uptr B = internal_mmap(NULL, RoundedSize, PROT_READ | PROT_WRITE,
+                         MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+  int ErrNo;
+  if (UNLIKELY(internal_iserror(B, &ErrNo))) {
     if (Verbosity())
-      Report("XRay Profiling: Failed to allocate memory of size %d.\n",
-             sizeof(T));
+      Report(
+          "XRay Profiling: Failed to allocate memory of size %d; Error = %d.\n",
+          RoundedSize, B);
     return nullptr;
   }
   return reinterpret_cast<T *>(B);
 }
 
 template <class T> void deallocate(T *B) XRAY_NEVER_INSTRUMENT {
   if (B == nullptr)
     return;
-  internal_munmap(B, sizeof(T));
+  uptr RoundedSize = RoundUpTo(sizeof(T), GetPageSizeCached());
+  internal_munmap(B, RoundedSize);
 }
 
 template <class T = uint8_t> T *allocateBuffer(size_t S) XRAY_NEVER_INSTRUMENT {
-  auto B = reinterpret_cast<void *>(
-      internal_mmap(NULL, S * sizeof(T), PROT_READ | PROT_WRITE,
-                    MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
-  if (B == MAP_FAILED) {
+  uptr RoundedSize = RoundUpTo(S * sizeof(T), GetPageSizeCached());
+  uptr B = internal_mmap(NULL, RoundedSize, PROT_READ | PROT_WRITE,
+                         MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+  int ErrNo;
+  if (UNLIKELY(internal_iserror(B, &ErrNo))) {
     if (Verbosity())
-      Report("XRay Profiling: Failed to allocate memory of size %d.\n", S);
+      Report(
+          "XRay Profiling: Failed to allocate memory of size %d; Error = %d.\n",
+          RoundedSize, B);
     return nullptr;
   }
   return reinterpret_cast<T *>(B);
 }
 
 template <class T> void deallocateBuffer(T *B, size_t S) XRAY_NEVER_INSTRUMENT {
   if (B == nullptr)
     return;
-  internal_munmap(B, S);
+  uptr RoundedSize = RoundUpTo(S * sizeof(T), GetPageSizeCached());
+  internal_munmap(B, RoundedSize);
 }
 
 template <class T, class... U>
 T *initArray(size_t N, U &&... Us) XRAY_NEVER_INSTRUMENT {
   auto A = allocateBuffer<T>(N);
   if (A != nullptr)
     while (N > 0)
       new (A + (--N)) T(std::forward<U>(Us)...);
template <size_t N> struct Allocator {
   struct Block {
     /// Compute the minimum cache-line size multiple that is >= N.
     static constexpr auto Size = nearest_boundary(N, kCacheLineSize);
     void *Data;
   };
 
 private:
   const size_t MaxMemory{0};
-  void *BackingStore = nullptr;
-  void *AlignedNextBlock = nullptr;
+  uint8_t *BackingStore = nullptr;
+  uint8_t *AlignedNextBlock = nullptr;
   size_t AllocatedBlocks = 0;
   SpinMutex Mutex{};
 
   void *Alloc() XRAY_NEVER_INSTRUMENT {
     SpinMutexLock Lock(&Mutex);
     if (UNLIKELY(BackingStore == nullptr)) {
-      BackingStore = reinterpret_cast<void *>(
-          internal_mmap(NULL, MaxMemory, PROT_READ | PROT_WRITE,
-                        MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
-      if (BackingStore == MAP_FAILED) {
-        BackingStore = nullptr;
+      BackingStore = allocateBuffer(MaxMemory);
+      if (BackingStore == nullptr) {
         if (Verbosity())
           Report("XRay Profiling: Failed to allocate memory for allocator.\n");
         return nullptr;
       }
 
       AlignedNextBlock = BackingStore;
 
       // Ensure that NextBlock is aligned appropriately.
       auto BackingStoreNum = reinterpret_cast<uintptr_t>(BackingStore);
       auto AlignedNextBlockNum = nearest_boundary(
           reinterpret_cast<uintptr_t>(AlignedNextBlock), kCacheLineSize);
       if (diff(AlignedNextBlockNum, BackingStoreNum) > ptrdiff_t(MaxMemory)) {
-        munmap(BackingStore, MaxMemory);
+        deallocateBuffer(BackingStore, MaxMemory);
         AlignedNextBlock = BackingStore = nullptr;
         if (Verbosity())
           Report("XRay Profiling: Cannot obtain enough memory from "
                  "preallocated region.\n");
         return nullptr;
       }
 
-      AlignedNextBlock = reinterpret_cast<void *>(AlignedNextBlockNum);
+      AlignedNextBlock = reinterpret_cast<uint8_t *>(AlignedNextBlockNum);
 
       // Assert that AlignedNextBlock is cache-line aligned.
       DCHECK_EQ(reinterpret_cast<uintptr_t>(AlignedNextBlock) % kCacheLineSize,
                 0);
     }
 
     if ((AllocatedBlocks * Block::Size) >= MaxMemory)
       return nullptr;
 
     // Align the pointer we'd like to return to an appropriate alignment, then
     // advance the pointer from where to start allocations.
     void *Result = AlignedNextBlock;
-    AlignedNextBlock = reinterpret_cast<void *>(
-        reinterpret_cast<char *>(AlignedNextBlock) + N);
+    AlignedNextBlock = reinterpret_cast<uint8_t *>(
+        reinterpret_cast<uint8_t *>(AlignedNextBlock) + N);
     ++AllocatedBlocks;
     return Result;
   }
 
 public:
   explicit Allocator(size_t M) XRAY_NEVER_INSTRUMENT
       : MaxMemory(nearest_boundary(M, kCacheLineSize)) {}
 
   Block Allocate() XRAY_NEVER_INSTRUMENT { return {Alloc()}; }
 
   ~Allocator() NOEXCEPT XRAY_NEVER_INSTRUMENT {
     if (BackingStore != nullptr) {
-      internal_munmap(BackingStore, MaxMemory);
+      deallocateBuffer(BackingStore, MaxMemory);
     }
   }
 };
 
 } // namespace __xray
 
 #endif // XRAY_ALLOCATOR_H

compiler-rt/trunk/lib/xray/xray_buffer_queue.h

 #ifndef XRAY_BUFFER_QUEUE_H
 #define XRAY_BUFFER_QUEUE_H
 
 #include "sanitizer_common/sanitizer_atomic.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_mutex.h"
 #include "xray_defs.h"
 #include <cstddef>
+#include <cstdint>
 
 namespace __xray {
 
 /// BufferQueue implements a circular queue of fixed sized buffers (much like a
 /// freelist) but is concerned mostly with making it really quick to initialise,
 /// finalise, and get/return buffers to the queue. This is one key component of
 /// the "flight data recorder" (FDR) mode to support ongoing XRay function call
 /// trace collection.
private:
   // Amount of pre-allocated buffers.
   size_t BufferCount;
 
   SpinMutex Mutex;
   atomic_uint8_t Finalizing;
 
   // A pointer to a contiguous block of memory to serve as the backing store for
   // all the individual buffers handed out.
-  void *BackingStore;
+  uint8_t *BackingStore;
 
   // A dynamically allocated array of BufferRep instances.
   BufferRep *Buffers;
 
   // Pointer to the next buffer to be handed out.
   BufferRep *Next;
 
   // Pointer to the entry in the array where the next released buffer will be

compiler-rt/trunk/lib/xray/xray_fdr_logging.cc

XRayBuffer fdrIterator(const XRayBuffer B) {
   auto &Header = reinterpret_cast<XRayFileHeader &>(HeaderStorage);
   if (B.Data == nullptr && B.Size == 0) {
     Header.FdrData = FdrAdditionalHeaderData{BQ->ConfiguredBufferSize()};
     return XRayBuffer{static_cast<void *>(&Header), sizeof(Header)};
   }
 
   static BufferQueue::const_iterator It{};
   static BufferQueue::const_iterator End{};
-  static void *CurrentBuffer{nullptr};
+  static uint8_t *CurrentBuffer{nullptr};
   static size_t SerializedBufferSize = 0;
   if (B.Data == static_cast<void *>(&Header) && B.Size == sizeof(Header)) {
     // From this point on, we provide raw access to the raw buffer we're getting
     // from the BufferQueue. We're relying on the iterators from the current
     // Buffer queue.
     It = BQ->cbegin();
     End = BQ->cend();
   }

compiler-rt/trunk/lib/xray/xray_profile_collector.cc

while (!DFSStack.empty()) {
         DFSStack.Append(C.NodePtr);
     }
   }
 }
 
 static void serializeRecords(ProfileBuffer *Buffer, const BlockHeader &Header,
                              const ProfileRecordArray &ProfileRecords)
     XRAY_NEVER_INSTRUMENT {
-  auto NextPtr = static_cast<char *>(
+  auto NextPtr = static_cast<uint8_t *>(
                      internal_memcpy(Buffer->Data, &Header, sizeof(Header))) +
                  sizeof(Header);
   for (const auto &Record : ProfileRecords) {
     // List of IDs follow:
     for (const auto FId : Record.Path)
       NextPtr =
-          static_cast<char *>(internal_memcpy(NextPtr, &FId, sizeof(FId))) +
+          static_cast<uint8_t *>(internal_memcpy(NextPtr, &FId, sizeof(FId))) +
           sizeof(FId);
 
     // Add the sentinel here.
     constexpr int32_t SentinelFId = 0;
-    NextPtr = static_cast<char *>(
+    NextPtr = static_cast<uint8_t *>(
                   internal_memset(NextPtr, SentinelFId, sizeof(SentinelFId))) +
               sizeof(SentinelFId);
 
     // Add the node data here.
     NextPtr =
-        static_cast<char *>(internal_memcpy(NextPtr, &Record.Node->CallCount,
-                                            sizeof(Record.Node->CallCount))) +
+        static_cast<uint8_t *>(internal_memcpy(
+            NextPtr, &Record.Node->CallCount, sizeof(Record.Node->CallCount))) +
         sizeof(Record.Node->CallCount);
-    NextPtr = static_cast<char *>(
+    NextPtr = static_cast<uint8_t *>(
                   internal_memcpy(NextPtr, &Record.Node->CumulativeLocalTime,
                                   sizeof(Record.Node->CumulativeLocalTime))) +
               sizeof(Record.Node->CumulativeLocalTime);
   }
 
-  DCHECK_EQ(NextPtr - static_cast<char *>(Buffer->Data), Buffer->Size);
+  DCHECK_EQ(NextPtr - static_cast<uint8_t *>(Buffer->Data), Buffer->Size);
 }
 
 } // namespace
 
 void serialize() XRAY_NEVER_INSTRUMENT {
   SpinMutexLock Lock(&GlobalMutex);
 
   if (GlobalAllocators == nullptr || ThreadTries == nullptr ||
       ProfileBuffers == nullptr)
     return;
 
   // Clear out the global ProfileBuffers, if it's not empty.
   for (auto &B : *ProfileBuffers)
-    deallocateBuffer(B.Data, B.Size);
+    deallocateBuffer(reinterpret_cast<uint8_t *>(B.Data), B.Size);
   ProfileBuffers->trim(ProfileBuffers->size());
 
   if (ThreadTries->empty())
     return;
 
   // Then repopulate the global ProfileBuffers.
   u32 I = 0;
   for (const auto &ThreadTrie : *ThreadTries) {

 }
 
 void reset() XRAY_NEVER_INSTRUMENT {
   SpinMutexLock Lock(&GlobalMutex);
 
   if (ProfileBuffers != nullptr) {
     // Clear out the profile buffers that have been serialized.
     for (auto &B : *ProfileBuffers)
-      deallocateBuffer(B.Data, B.Size);
+      deallocateBuffer(reinterpret_cast<uint8_t *>(B.Data), B.Size);
     ProfileBuffers->trim(ProfileBuffers->size());
   }
 
   if (ThreadTries != nullptr) {
     // Clear out the function call tries per thread.
     for (auto &T : *ThreadTries) {
       auto Trie = reinterpret_cast<FunctionCallTrie *>(&T.TrieStorage);
       Trie->~FunctionCallTrie();