Index: lib/xray/xray_buffer_queue.h =================================================================== --- lib/xray/xray_buffer_queue.h +++ lib/xray/xray_buffer_queue.h @@ -17,8 +17,8 @@ #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_mutex.h" -#include -#include +#include +#include #include namespace __xray { @@ -36,15 +36,23 @@ }; private: + // Size of each individual Buffer. size_t BufferSize; // We use a bool to indicate whether the Buffer has been used in this // freelist implementation. - std::deque> Buffers; - __sanitizer::BlockingMutex Mutex; - std::unordered_set OwnedBuffers; + std::unique_ptr[]> Buffers; + size_t BufferCount; + + __sanitizer::SpinMutex Mutex; __sanitizer::atomic_uint8_t Finalizing; + // Sorted buffer pointers, making it quick to find buffers that we own. + std::unique_ptr OwnedBuffers; + + std::tuple *Next; + std::tuple *First; + public: enum class ErrorCode : unsigned { Ok, @@ -117,8 +125,9 @@ /// Buffer is marked 'used' (i.e. has been the result of getBuffer(...) and a /// releaseBuffer(...) operation). template void apply(F Fn) { - __sanitizer::BlockingMutexLock G(&Mutex); - for (const auto &T : Buffers) { + __sanitizer::SpinMutexLock G(&Mutex); + for (auto I = Buffers.get(); I != Buffers.get() + BufferCount; ++I) { + const auto &T = *I; if (std::get<1>(T)) Fn(std::get<0>(T)); } Index: lib/xray/xray_buffer_queue.cc =================================================================== --- lib/xray/xray_buffer_queue.cc +++ lib/xray/xray_buffer_queue.cc @@ -16,6 +16,7 @@ #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_libc.h" +#include #include #include @@ -23,18 +24,20 @@ using namespace __sanitizer; BufferQueue::BufferQueue(std::size_t B, std::size_t N, bool &Success) - : BufferSize(B), Buffers(N), Mutex(), OwnedBuffers(), Finalizing{0} { - for (auto &T : Buffers) { + : BufferSize(B), Buffers(new std::tuple[N]()), BufferCount(N), + Mutex(), Finalizing{0}, OwnedBuffers(new void *[N]()), + Next(Buffers.get()), First(nullptr) { + for (size_t i = 0; i < BufferCount; ++i) { + auto &T = Buffers[i]; void *Tmp = malloc(BufferSize); if (Tmp == nullptr) { Success = false; return; } - auto &Buf = std::get<0>(T); Buf.Buffer = Tmp; Buf.Size = B; - OwnedBuffers.emplace(Tmp); + OwnedBuffers[i] = Tmp; } Success = true; } @@ -42,27 +45,41 @@ BufferQueue::ErrorCode BufferQueue::getBuffer(Buffer &Buf) { if (__sanitizer::atomic_load(&Finalizing, __sanitizer::memory_order_acquire)) return ErrorCode::QueueFinalizing; - __sanitizer::BlockingMutexLock Guard(&Mutex); - if (Buffers.empty()) + __sanitizer::SpinMutexLock Guard(&Mutex); + + if (First != nullptr && Next == First) return ErrorCode::NotEnoughMemory; - auto &T = Buffers.front(); + auto &T = *Next; auto &B = std::get<0>(T); Buf = B; - B.Buffer = nullptr; - B.Size = 0; - Buffers.pop_front(); + std::get<1>(T) = true; + if (First == nullptr) + First = Next; + ++Next; + if (Next == Buffers.get() + BufferCount) + Next = Buffers.get(); return ErrorCode::Ok; } BufferQueue::ErrorCode BufferQueue::releaseBuffer(Buffer &Buf) { - if (OwnedBuffers.count(Buf.Buffer) == 0) + // Blitz through the buffers array to find the buffer. + if (!std::any_of(OwnedBuffers.get(), OwnedBuffers.get() + BufferCount, + [&Buf](void *P) { return P == Buf.Buffer; })) return ErrorCode::UnrecognizedBuffer; - __sanitizer::BlockingMutexLock Guard(&Mutex); + __sanitizer::SpinMutexLock Guard(&Mutex); + + // This points to a semantic bug, we really ought to not be releasing more + // buffers than we actually get. + if (First == nullptr || First == Next) + return ErrorCode::NotEnoughMemory; // Now that the buffer has been released, we mark it as "used". - Buffers.emplace(Buffers.end(), Buf, true /* used */); + *First = std::make_tuple(Buf, true); Buf.Buffer = nullptr; Buf.Size = 0; + ++First; + if (First == Buffers.get() + BufferCount) + First = Buffers.get(); return ErrorCode::Ok; } @@ -74,7 +91,8 @@ } BufferQueue::~BufferQueue() { - for (auto &T : Buffers) { + for (auto I = Buffers.get(); I != Buffers.get() + BufferCount; ++I) { + auto &T = *I; auto &Buf = std::get<0>(T); free(Buf.Buffer); }