Index: compiler-rt/lib/xray/xray_buffer_queue.cc =================================================================== --- compiler-rt/lib/xray/xray_buffer_queue.cc +++ compiler-rt/lib/xray/xray_buffer_queue.cc @@ -13,17 +13,50 @@ // //===----------------------------------------------------------------------===// #include "xray_buffer_queue.h" -#include "sanitizer_common/sanitizer_allocator_internal.h" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_posix.h" #include +#include + +#ifndef MAP_NORESERVE +// no-op on NetBSD (at least), unsupported flag on FreeBSD +#define MAP_NORESERVE 0 +#endif using namespace __xray; using namespace __sanitizer; +template static T *allocRaw(size_t N) { + // TODO: Report errors? + // We use MAP_NORESERVE on platforms where it's supported to ensure that the + // pages we're allocating for XRay never end up in pages that can be swapped + // in/out. We're doing this because for FDR mode, we want to ensure that + // writes to the buffers stay resident in memory to prevent XRay itself from + // causing swapping/thrashing. + // + // In the case when XRay pages cannot be swapped in/out or there's not enough + // RAM to back these pages, we're willing to cause a segmentation fault + // instead of introducing latency in the measurement. We assume here that + // there are enough pages that are swappable in/out outside of the buffers + // being used by FDR mode (which are bounded and configurable anyway) to allow + // us to keep using always-resident memory. + // + // TODO: Make this configurable? + void *A = reinterpret_cast( + internal_mmap(NULL, N * sizeof(T), PROT_WRITE | PROT_READ, + MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE, -1, 0)); + return (A == MAP_FAILED) ? nullptr : reinterpret_cast(A); +} + +template static void deallocRaw(T *ptr, size_t N) { + // TODO: Report errors? + if (ptr != nullptr) + internal_munmap(ptr, N); +} + template static T *initArray(size_t N) { - auto A = reinterpret_cast( - InternalAlloc(N * sizeof(T), nullptr, kCacheLineSize)); + auto A = allocRaw(N); if (A != nullptr) while (N > 0) new (A + (--N)) T(); @@ -42,19 +75,19 @@ // Clean up the buffers we've already allocated. for (auto B = Buffers, E = Buffers + BufferCount; B != E; ++B) B->~BufferRep(); - InternalFree(Buffers); + deallocRaw(Buffers, N); Success = false; return; }; for (size_t i = 0; i < N; ++i) { auto &T = Buffers[i]; - void *Tmp = InternalAlloc(BufferSize, nullptr, 64); + void *Tmp = allocRaw(BufferSize); if (Tmp == nullptr) { Success = false; return; } - void *Extents = InternalAlloc(sizeof(BufferExtents), nullptr, 64); + auto *Extents = allocRaw(1); if (Extents == nullptr) { Success = false; return; @@ -62,7 +95,7 @@ auto &Buf = T.Buff; Buf.Data = Tmp; Buf.Size = B; - Buf.Extents = reinterpret_cast(Extents); + Buf.Extents = Extents; OwnedBuffers[i] = Tmp; } Success = true; @@ -128,11 +161,11 @@ for (auto I = Buffers, E = Buffers + BufferCount; I != E; ++I) { auto &T = *I; auto &Buf = T.Buff; - InternalFree(Buf.Data); - InternalFree(Buf.Extents); + deallocRaw(Buf.Data, Buf.Size); + deallocRaw(Buf.Extents, 1); } for (auto B = Buffers, E = Buffers + BufferCount; B != E; ++B) B->~BufferRep(); - InternalFree(Buffers); - InternalFree(OwnedBuffers); + deallocRaw(Buffers, BufferCount); + deallocRaw(OwnedBuffers, BufferCount); }