[XRay] XRAY_NEVER_INSTRUMENT more functions, consolidate allocators

[dberris]

Summary

In this change we apply XRAY_NEVER_INSTRUMENT to more functions in the
profiling implementation to ensure that these never get instrumented if
the compiler used to build the library is capable of doing XRay
instrumentation.

We also consolidate all the allocators into a single header
(xray_allocator.h) which sidestep the use of the internal allocator
implementation in sanitizer_common.

This addresses more cases mentioned in llvm.org/PR38577.

[mboerger]

LGTM

lib/xray/xray_allocator.h

 //===----------------------------------------------------------------------===//
 #ifndef XRAY_ALLOCATOR_H
 #define XRAY_ALLOCATOR_H
 
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_internal_defs.h"
 #include "sanitizer_common/sanitizer_mutex.h"
 #include "sanitizer_common/sanitizer_posix.h"
+#include "xray_defs.h"
 #include "xray_utils.h"
 #include <cstddef>
 #include <cstdint>
 #include <sys/mman.h>
 
 namespace __xray {
 
+// We implement our own memory allocation routine which will bypass the
+// internal allocator. This allows us to manage the memory directly, using
+// mmap'ed memory to back the allocators.
+template <class T> T *allocate() XRAY_NEVER_INSTRUMENT {
+  auto B = reinterpret_cast<void *>(
+      internal_mmap(NULL, sizeof(T), PROT_READ | PROT_WRITE,
+                    MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
+  if (B == MAP_FAILED) {
+    if (Verbosity())
+      Report("XRay Profiling: Failed to allocate memory of size %d.\n",
+             sizeof(T));
+    return nullptr;
+  }
+  return reinterpret_cast<T *>(B);
+}
+
+template <class T> void deallocate(T *B) XRAY_NEVER_INSTRUMENT {
+  if (B == nullptr)
+    return;
+  internal_munmap(B, sizeof(T));
+}
+
+inline void *allocateBuffer(size_t S) XRAY_NEVER_INSTRUMENT {
+  auto B = reinterpret_cast<void *>(internal_mmap(
+      NULL, S, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
+  if (B == MAP_FAILED) {
+    if (Verbosity())
+      Report("XRay Profiling: Failed to allocate memory of size %d.\n", S);
+    return nullptr;
+  }
+  return B;
+}
+
+inline void deallocateBuffer(void *B, size_t S) XRAY_NEVER_INSTRUMENT {
+  if (B == nullptr)
+    return;
+  internal_munmap(B, S);
+}
+
 /// The Allocator type hands out fixed-sized chunks of memory that are
 /// cache-line aligned and sized. This is useful for placement of
 /// performance-sensitive data in memory that's frequently accessed. The
 /// allocator also self-limits the peak memory usage to a dynamically defined
 /// maximum.
 ///
 /// N is the lower-bound size of the block of memory to return from the
 /// allocation function. N is used to compute the size of a block, which is

 
 private:
   const size_t MaxMemory{0};
   void *BackingStore = nullptr;
   void *AlignedNextBlock = nullptr;
   size_t AllocatedBlocks = 0;
   SpinMutex Mutex{};
 
-  void *Alloc() {
+  void *Alloc() XRAY_NEVER_INSTRUMENT {
     SpinMutexLock Lock(&Mutex);
     if (UNLIKELY(BackingStore == nullptr)) {
       BackingStore = reinterpret_cast<void *>(
           internal_mmap(NULL, MaxMemory, PROT_READ | PROT_WRITE,
-                        MAP_PRIVATE | MAP_ANONYMOUS, 0, 0));
+                        MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
       if (BackingStore == MAP_FAILED) {
         BackingStore = nullptr;
         if (Verbosity())
           Report("XRay Profiling: Failed to allocate memory for allocator.\n");
         return nullptr;
       }
 
       AlignedNextBlock = BackingStore;
void *Alloc() XRAY_NEVER_INSTRUMENT {
     void *Result = AlignedNextBlock;
     AlignedNextBlock = reinterpret_cast<void *>(
         reinterpret_cast<char *>(AlignedNextBlock) + N);
     ++AllocatedBlocks;
     return Result;
   }
 
 public:
-  explicit Allocator(size_t M)
+  explicit Allocator(size_t M) XRAY_NEVER_INSTRUMENT
       : MaxMemory(nearest_boundary(M, kCacheLineSize)) {}
 
-  Block Allocate() { return {Alloc()}; }
+  Block Allocate() XRAY_NEVER_INSTRUMENT { return {Alloc()}; }
 
-  ~Allocator() NOEXCEPT {
+  ~Allocator() NOEXCEPT XRAY_NEVER_INSTRUMENT {
     if (BackingStore != nullptr) {
       internal_munmap(BackingStore, MaxMemory);
     }
   }
 };
 
 } // namespace __xray
 
 #endif // XRAY_ALLOCATOR_H

lib/xray/xray_function_call_trie.h

 // This file is a part of XRay, a dynamic runtime instrumentation system.
 //
 // This file defines the interface for a function call trie.
 //
 //===----------------------------------------------------------------------===//
 #ifndef XRAY_FUNCTION_CALL_TRIE_H
 #define XRAY_FUNCTION_CALL_TRIE_H
 
-#include "sanitizer_common/sanitizer_allocator_internal.h"
+#include "xray_defs.h"
 #include "xray_profiling_flags.h"
 #include "xray_segmented_array.h"
 #include <memory> // For placement new.
 #include <utility>
 
 namespace __xray {
 
 /// A FunctionCallTrie represents the stack traces of XRay instrumented
struct Node {
     NodeIdPairArray Callees;
     int64_t CallCount;
     int64_t CumulativeLocalTime; // Typically in TSC deltas, not wall-time.
     int32_t FId;
 
     // We add a constructor here to allow us to inplace-construct through
     // Array<...>'s AppendEmplace.
     Node(Node *P, NodeIdPairAllocatorType &A, int64_t CC, int64_t CLT,
-         int32_t F)
-        : Parent(P), Callees(A), CallCount(CC), CumulativeLocalTime(CLT),
+         int32_t F) XRAY_NEVER_INSTRUMENT : Parent(P),
+                                            Callees(A),
+                                            CallCount(CC),
+                                            CumulativeLocalTime(CLT),
                                             FId(F) {}
 
     // TODO: Include the compact histogram.
   };
 
 private:
   struct ShadowStackEntry {
     uint64_t EntryTSC;
     Node *NodePtr;
 
     // We add a constructor here to allow us to inplace-construct through
     // Array<...>'s AppendEmplace.
-    ShadowStackEntry(uint64_t T, Node *N) : EntryTSC{T}, NodePtr{N} {}
+    ShadowStackEntry(uint64_t T, Node *N) XRAY_NEVER_INSTRUMENT : EntryTSC{T},
+                                                                  NodePtr{N} {}
   };
 
   using NodeArray = Array<Node>;
   using RootArray = Array<Node *>;
   using ShadowStackArray = Array<ShadowStackEntry>;
 
 public:
   // We collate the allocators we need into a single struct, as a convenience to
   // allow us to initialize these as a group.
   struct Allocators {
     using NodeAllocatorType = NodeArray::AllocatorType;
     using RootAllocatorType = RootArray::AllocatorType;
     using ShadowStackAllocatorType = ShadowStackArray::AllocatorType;
 
     NodeAllocatorType *NodeAllocator = nullptr;
     RootAllocatorType *RootAllocator = nullptr;
     ShadowStackAllocatorType *ShadowStackAllocator = nullptr;
     NodeIdPairAllocatorType *NodeIdPairAllocator = nullptr;
 
     Allocators() {}
     Allocators(const Allocators &) = delete;
     Allocators &operator=(const Allocators &) = delete;
 
-    Allocators(Allocators &&O)
-        : NodeAllocator(O.NodeAllocator), RootAllocator(O.RootAllocator),
+    Allocators(Allocators &&O) XRAY_NEVER_INSTRUMENT
+        : NodeAllocator(O.NodeAllocator),
+          RootAllocator(O.RootAllocator),
           ShadowStackAllocator(O.ShadowStackAllocator),
           NodeIdPairAllocator(O.NodeIdPairAllocator) {
       O.NodeAllocator = nullptr;
       O.RootAllocator = nullptr;
       O.ShadowStackAllocator = nullptr;
       O.NodeIdPairAllocator = nullptr;
     }
 
-    Allocators &operator=(Allocators &&O) {
+    Allocators &operator=(Allocators &&O) XRAY_NEVER_INSTRUMENT {
       {
         auto Tmp = O.NodeAllocator;
         O.NodeAllocator = this->NodeAllocator;
         this->NodeAllocator = Tmp;
       }
       {
         auto Tmp = O.RootAllocator;
         O.RootAllocator = this->RootAllocator;
         this->RootAllocator = Tmp;
       }
       {
         auto Tmp = O.ShadowStackAllocator;
         O.ShadowStackAllocator = this->ShadowStackAllocator;
         this->ShadowStackAllocator = Tmp;
       }
       {
         auto Tmp = O.NodeIdPairAllocator;
         O.NodeIdPairAllocator = this->NodeIdPairAllocator;
         this->NodeIdPairAllocator = Tmp;
       }
       return *this;
     }
 
-    ~Allocators() {
+    ~Allocators() XRAY_NEVER_INSTRUMENT {
       // Note that we cannot use delete on these pointers, as they need to be
       // returned to the sanitizer_common library's internal memory tracking
       // system.
       if (NodeAllocator != nullptr) {
         NodeAllocator->~NodeAllocatorType();
-        InternalFree(NodeAllocator);
+        deallocate(NodeAllocator);
         NodeAllocator = nullptr;
       }
       if (RootAllocator != nullptr) {
         RootAllocator->~RootAllocatorType();
-        InternalFree(RootAllocator);
+        deallocate(RootAllocator);
         RootAllocator = nullptr;
       }
       if (ShadowStackAllocator != nullptr) {
         ShadowStackAllocator->~ShadowStackAllocatorType();
-        InternalFree(ShadowStackAllocator);
+        deallocate(ShadowStackAllocator);
         ShadowStackAllocator = nullptr;
       }
       if (NodeIdPairAllocator != nullptr) {
         NodeIdPairAllocator->~NodeIdPairAllocatorType();
-        InternalFree(NodeIdPairAllocator);
+        deallocate(NodeIdPairAllocator);
         NodeIdPairAllocator = nullptr;
       }
     }
   };
 
   // TODO: Support configuration of options through the arguments.
-  static Allocators InitAllocators() {
+  static Allocators InitAllocators() XRAY_NEVER_INSTRUMENT {
     return InitAllocatorsCustom(profilingFlags()->per_thread_allocator_max);
   }
 
-  static Allocators InitAllocatorsCustom(uptr Max) {
+  static Allocators InitAllocatorsCustom(uptr Max) XRAY_NEVER_INSTRUMENT {
     Allocators A;
-    auto NodeAllocator = reinterpret_cast<Allocators::NodeAllocatorType *>(
-        InternalAlloc(sizeof(Allocators::NodeAllocatorType)));
+    auto NodeAllocator = allocate<Allocators::NodeAllocatorType>();
     new (NodeAllocator) Allocators::NodeAllocatorType(Max);
     A.NodeAllocator = NodeAllocator;
 
-    auto RootAllocator = reinterpret_cast<Allocators::RootAllocatorType *>(
-        InternalAlloc(sizeof(Allocators::RootAllocatorType)));
+    auto RootAllocator = allocate<Allocators::RootAllocatorType>();
     new (RootAllocator) Allocators::RootAllocatorType(Max);
     A.RootAllocator = RootAllocator;
 
     auto ShadowStackAllocator =
-        reinterpret_cast<Allocators::ShadowStackAllocatorType *>(
-            InternalAlloc(sizeof(Allocators::ShadowStackAllocatorType)));
+        allocate<Allocators::ShadowStackAllocatorType>();
     new (ShadowStackAllocator) Allocators::ShadowStackAllocatorType(Max);
     A.ShadowStackAllocator = ShadowStackAllocator;
 
-    auto NodeIdPairAllocator = reinterpret_cast<NodeIdPairAllocatorType *>(
-        InternalAlloc(sizeof(NodeIdPairAllocatorType)));
+    auto NodeIdPairAllocator = allocate<NodeIdPairAllocatorType>();
     new (NodeIdPairAllocator) NodeIdPairAllocatorType(Max);
     A.NodeIdPairAllocator = NodeIdPairAllocator;
     return A;
   }
 
 private:
   NodeArray Nodes;
   RootArray Roots;
   ShadowStackArray ShadowStack;
   NodeIdPairAllocatorType *NodeIdPairAllocator = nullptr;
 
 public:
-  explicit FunctionCallTrie(const Allocators &A)
-      : Nodes(*A.NodeAllocator), Roots(*A.RootAllocator),
+  explicit FunctionCallTrie(const Allocators &A) XRAY_NEVER_INSTRUMENT
+      : Nodes(*A.NodeAllocator),
+        Roots(*A.RootAllocator),
         ShadowStack(*A.ShadowStackAllocator),
         NodeIdPairAllocator(A.NodeIdPairAllocator) {}
 
-  void enterFunction(const int32_t FId, uint64_t TSC) {
+  void enterFunction(const int32_t FId, uint64_t TSC) XRAY_NEVER_INSTRUMENT {
     DCHECK_NE(FId, 0);
     // This function primarily deals with ensuring that the ShadowStack is
     // consistent and ready for when an exit event is encountered.
     if (UNLIKELY(ShadowStack.empty())) {
       auto NewRoot =
           Nodes.AppendEmplace(nullptr, *NodeIdPairAllocator, 0, 0, FId);
       if (UNLIKELY(NewRoot == nullptr))
         return;
if (UNLIKELY(NewNode == nullptr))
       return;
     DCHECK_NE(NewNode, nullptr);
     TopNode->Callees.AppendEmplace(NewNode, FId);
     ShadowStack.AppendEmplace(TSC, NewNode);
     DCHECK_NE(ShadowStack.back().NodePtr, nullptr);
     return;
   }
 
-  void exitFunction(int32_t FId, uint64_t TSC) {
+  void exitFunction(int32_t FId, uint64_t TSC) XRAY_NEVER_INSTRUMENT {
     // When we exit a function, we look up the ShadowStack to see whether we've
     // entered this function before. We do as little processing here as we can,
     // since most of the hard work would have already been done at function
     // entry.
     uint64_t CumulativeTreeTime = 0;
     while (!ShadowStack.empty()) {
       const auto &Top = ShadowStack.back();
       auto TopNode = Top.NodePtr;
       DCHECK_NE(TopNode, nullptr);
       auto LocalTime = TSC - Top.EntryTSC;
       TopNode->CallCount++;
       TopNode->CumulativeLocalTime += LocalTime - CumulativeTreeTime;
       CumulativeTreeTime += LocalTime;
       ShadowStack.trim(1);
 
       // TODO: Update the histogram for the node.
       if (TopNode->FId == FId)
         break;
     }
   }
 
-  const RootArray &getRoots() const { return Roots; }
+  const RootArray &getRoots() const XRAY_NEVER_INSTRUMENT { return Roots; }
 
   // The deepCopyInto operation will update the provided FunctionCallTrie by
   // re-creating the contents of this particular FunctionCallTrie in the other
   // FunctionCallTrie. It will do this using a Depth First Traversal from the
   // roots, and while doing so recreating the traversal in the provided
   // FunctionCallTrie.
   //
   // This operation will *not* destroy the state in `O`, and thus may cause some
   // duplicate entries in `O` if it is not empty.
   //
   // This function is *not* thread-safe, and may require external
   // synchronisation of both "this" and |O|.
   //
   // This function must *not* be called with a non-empty FunctionCallTrie |O|.
-  void deepCopyInto(FunctionCallTrie &O) const {
+  void deepCopyInto(FunctionCallTrie &O) const XRAY_NEVER_INSTRUMENT {
     DCHECK(O.getRoots().empty());
 
     // We then push the root into a stack, to use as the parent marker for new
     // nodes we push in as we're traversing depth-first down the call tree.
     struct NodeAndParent {
       FunctionCallTrie::Node *Node;
       FunctionCallTrie::Node *NewNode;
     };
public:
   // The mergeInto operation will update the provided FunctionCallTrie by
   // traversing the current trie's roots and updating (i.e. merging) the data in
   // the nodes with the data in the target's nodes. If the node doesn't exist in
   // the provided trie, we add a new one in the right position, and inherit the
   // data from the original (current) trie, along with all its callees.
   //
   // This function is *not* thread-safe, and may require external
   // synchronisation of both "this" and |O|.
-  void mergeInto(FunctionCallTrie &O) const {
+  void mergeInto(FunctionCallTrie &O) const XRAY_NEVER_INSTRUMENT {
     struct NodeAndTarget {
       FunctionCallTrie::Node *OrigNode;
       FunctionCallTrie::Node *TargetNode;
     };
     using Stack = Array<NodeAndTarget>;
     typename Stack::AllocatorType StackAllocator(
         profilingFlags()->stack_allocator_max);
     Stack DFSStack(StackAllocator);

lib/xray/xray_profile_collector.cc

 // This file is a part of XRay, a dynamic runtime instrumentation system.
 //
 // This implements the interface for the profileCollectorService.
 //
 //===----------------------------------------------------------------------===//
 #include "xray_profile_collector.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "xray_allocator.h"
+#include "xray_defs.h"
 #include "xray_profiling_flags.h"
 #include "xray_segmented_array.h"
 #include <memory>
 #include <pthread.h>
 #include <utility>
 
 namespace __xray {
 namespace profileCollectorService {
static typename std::aligned_storage<sizeof(ProfileBufferArrayAllocator)>::type
     ProfileBufferArrayAllocatorStorage;
 
 static ThreadTriesArray *ThreadTries = nullptr;
 static ThreadTriesArrayAllocator *ThreadTriesAllocator = nullptr;
 static ProfileBufferArray *ProfileBuffers = nullptr;
 static ProfileBufferArrayAllocator *ProfileBuffersAllocator = nullptr;
 static FunctionCallTrie::Allocators *GlobalAllocators = nullptr;
 
-static void *allocateBuffer(size_t S) {
-  auto B = reinterpret_cast<void *>(internal_mmap(
-      NULL, S, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
-  if (B == MAP_FAILED) {
-    if (Verbosity())
-      Report("XRay Profiling: Failed to allocate memory of size %d.\n", S);
-    return nullptr;
-  }
-  return B;
-}
-
-static void deallocateBuffer(void *B, size_t S) {
-  if (B == nullptr)
-    return;
-  internal_munmap(B, S);
-}
-
 } // namespace
 
-void post(const FunctionCallTrie &T, tid_t TId) {
+void post(const FunctionCallTrie &T, tid_t TId) XRAY_NEVER_INSTRUMENT {
   static pthread_once_t Once = PTHREAD_ONCE_INIT;
   pthread_once(&Once, +[] { reset(); });
 
   ThreadTrie *Item = nullptr;
   {
     SpinMutexLock Lock(&GlobalMutex);
     if (GlobalAllocators == nullptr || ThreadTries == nullptr)
       return;
struct ProfileRecord {
   using PathAllocator = typename PathArray::AllocatorType;
 
   // The Path in this record is the function id's from the leaf to the root of
   // the function call stack as represented from a FunctionCallTrie.
   PathArray Path;
   const FunctionCallTrie::Node *Node = nullptr;
 
   // Constructor for in-place construction.
-  ProfileRecord(PathAllocator &A, const FunctionCallTrie::Node *N)
-      : Path(A), Node(N) {}
+  ProfileRecord(PathAllocator &A,
+                const FunctionCallTrie::Node *N) XRAY_NEVER_INSTRUMENT
+      : Path(A),
+        Node(N) {}
 };
 
 namespace {
 
 using ProfileRecordArray = Array<ProfileRecord>;
 
 // Walk a depth-first traversal of each root of the FunctionCallTrie to generate
 // the path(s) and the data associated with the path.
-static void populateRecords(ProfileRecordArray &PRs,
-                            ProfileRecord::PathAllocator &PA,
-                            const FunctionCallTrie &Trie) {
+static void
+populateRecords(ProfileRecordArray &PRs, ProfileRecord::PathAllocator &PA,
+                const FunctionCallTrie &Trie) XRAY_NEVER_INSTRUMENT {
   using StackArray = Array<const FunctionCallTrie::Node *>;
   using StackAllocator = typename StackArray::AllocatorType;
   StackAllocator StackAlloc(profilingFlags()->stack_allocator_max);
   StackArray DFSStack(StackAlloc);
   for (const auto R : Trie.getRoots()) {
     DFSStack.Append(R);
     while (!DFSStack.empty()) {
       auto Node = DFSStack.back();
while (!DFSStack.empty()) {
 
       for (const auto C : Node->Callees)
         DFSStack.Append(C.NodePtr);
     }
   }
 }
 
 static void serializeRecords(ProfileBuffer *Buffer, const BlockHeader &Header,
-                             const ProfileRecordArray &ProfileRecords) {
+                             const ProfileRecordArray &ProfileRecords)
+    XRAY_NEVER_INSTRUMENT {
   auto NextPtr = static_cast<char *>(
                      internal_memcpy(Buffer->Data, &Header, sizeof(Header))) +
                  sizeof(Header);
   for (const auto &Record : ProfileRecords) {
     // List of IDs follow:
     for (const auto FId : Record.Path)
       NextPtr =
           static_cast<char *>(internal_memcpy(NextPtr, &FId, sizeof(FId))) +
NextPtr = static_cast<char *>(
               sizeof(Record.Node->CumulativeLocalTime);
   }
 
   DCHECK_EQ(NextPtr - static_cast<char *>(Buffer->Data), Buffer->Size);
 }
 
 } // namespace
 
-void serialize() {
+void serialize() XRAY_NEVER_INSTRUMENT {
   SpinMutexLock Lock(&GlobalMutex);
 
   if (GlobalAllocators == nullptr || ThreadTries == nullptr ||
       ProfileBuffers == nullptr)
     return;
 
   // Clear out the global ProfileBuffers, if it's not empty.
   for (auto &B : *ProfileBuffers)
for (const auto &ThreadTrie : *ThreadTries) {
     auto Buffer = ProfileBuffers->Append({});
     Buffer->Size = sizeof(Header) + CumulativeSizes;
     Buffer->Data = allocateBuffer(Buffer->Size);
     DCHECK_NE(Buffer->Data, nullptr);
     serializeRecords(Buffer, Header, ProfileRecords);
   }
 }
 
-void reset() {
+void reset() XRAY_NEVER_INSTRUMENT {
   SpinMutexLock Lock(&GlobalMutex);
 
   if (ProfileBuffers != nullptr) {
     // Clear out the profile buffers that have been serialized.
     for (auto &B : *ProfileBuffers)
       deallocateBuffer(B.Data, B.Size);
     ProfileBuffers->trim(ProfileBuffers->size());
   }
ProfileBuffersAllocator = reinterpret_cast<ProfileBufferArrayAllocator *>(
       &ProfileBufferArrayAllocatorStorage);
   new (ProfileBuffersAllocator)
       ProfileBufferArrayAllocator(profilingFlags()->global_allocator_max);
   ProfileBuffers =
       reinterpret_cast<ProfileBufferArray *>(&ProfileBuffersStorage);
   new (ProfileBuffers) ProfileBufferArray(*ProfileBuffersAllocator);
 }
 
-XRayBuffer nextBuffer(XRayBuffer B) {
+XRayBuffer nextBuffer(XRayBuffer B) XRAY_NEVER_INSTRUMENT {
   SpinMutexLock Lock(&GlobalMutex);
 
   if (ProfileBuffers == nullptr || ProfileBuffers->size() == 0)
     return {nullptr, 0};
 
   static pthread_once_t Once = PTHREAD_ONCE_INIT;
   static typename std::aligned_storage<sizeof(XRayProfilingFileHeader)>::type
       FileHeaderStorage;

lib/xray/xray_profiling.cc

 //===----------------------------------------------------------------------===//
 #include <memory>
 #include <time.h>
 
 #include "sanitizer_common/sanitizer_atomic.h"
 #include "sanitizer_common/sanitizer_flags.h"
 #include "xray/xray_interface.h"
 #include "xray/xray_log_interface.h"
-
 #include "xray_flags.h"
 #include "xray_profile_collector.h"
 #include "xray_profiling_flags.h"
 #include "xray_recursion_guard.h"
 #include "xray_tsc.h"
 #include "xray_utils.h"
 #include <pthread.h>
 
thread_local auto ThreadOnce = [] {
     auto &TLD = *reinterpret_cast<ProfilingData *>(&ThreadStorage);
     TLD.Allocators = Allocators;
     TLD.FCT = FCT;
     pthread_setspecific(ProfilingKey, &ThreadStorage);
     return false;
   }();
   (void)ThreadOnce;
 
-  auto &TLD = *reinterpret_cast<ProfilingData*>(&ThreadStorage);
+  auto &TLD = *reinterpret_cast<ProfilingData *>(&ThreadStorage);
 
   if (UNLIKELY(TLD.Allocators == nullptr || TLD.FCT == nullptr)) {
     auto *Allocators =
         reinterpret_cast<FunctionCallTrie::Allocators *>(&AllocatorsStorage);
     new (Allocators) FunctionCallTrie::Allocators();
     *Allocators = FunctionCallTrie::InitAllocators();
     auto *FCT = reinterpret_cast<FunctionCallTrie *>(&FunctionCallTrieStorage);
     new (FCT) FunctionCallTrie(*Allocators);
XRayLogFlushStatus profilingFlush() XRAY_NEVER_INSTRUMENT {
 
   return XRayLogFlushStatus::XRAY_LOG_FLUSHED;
 }
 
 namespace {
 
 thread_local atomic_uint8_t ReentranceGuard{0};
 
-static void postCurrentThreadFCT(ProfilingData &TLD) {
+static void postCurrentThreadFCT(ProfilingData &TLD) XRAY_NEVER_INSTRUMENT {
   if (TLD.Allocators == nullptr || TLD.FCT == nullptr)
     return;
 
+  if (!TLD.FCT->getRoots().empty())
     profileCollectorService::post(*TLD.FCT, GetTid());
+
   cleanupTLD();
 }
 
 } // namespace
 
 void profilingHandleArg0(int32_t FuncId,
                          XRayEntryType Entry) XRAY_NEVER_INSTRUMENT {
   unsigned char CPU;

lib/xray/xray_segmented_array.h

private:
   SegmentBase *Head = &SentinelSegment;
   SegmentBase *Tail = &SentinelSegment;
   size_t Size = 0;
 
   // Here we keep track of segments in the freelist, to allow us to re-use
   // segments when elements are trimmed off the end.
   SegmentBase *Freelist = &SentinelSegment;
 
-  Segment *NewSegment() {
+  Segment *NewSegment() XRAY_NEVER_INSTRUMENT {
     // We need to handle the case in which enough elements have been trimmed to
     // allow us to re-use segments we've allocated before. For this we look into
     // the Freelist, to see whether we need to actually allocate new blocks or
     // just re-use blocks we've already seen before.
     if (Freelist != &SentinelSegment) {
       auto *FreeSegment = Freelist;
       Freelist = FreeSegment->Next;
       FreeSegment->Next = &SentinelSegment;
       Freelist->Prev = &SentinelSegment;
       return static_cast<Segment *>(FreeSegment);
     }
 
     auto SegmentBlock = Alloc->Allocate();
     if (SegmentBlock.Data == nullptr)
       return nullptr;
 
     // Placement-new the Segment element at the beginning of the SegmentBlock.
     auto S = reinterpret_cast<Segment *>(SegmentBlock.Data);
     new (S) SegmentBase{&SentinelSegment, &SentinelSegment};
     return S;
   }
 
-  Segment *InitHeadAndTail() {
+  Segment *InitHeadAndTail() XRAY_NEVER_INSTRUMENT {
     DCHECK_EQ(Head, &SentinelSegment);
     DCHECK_EQ(Tail, &SentinelSegment);
     auto Segment = NewSegment();
     if (Segment == nullptr)
       return nullptr;
     DCHECK_EQ(Segment->Next, &SentinelSegment);
     DCHECK_EQ(Segment->Prev, &SentinelSegment);
     Head = Tail = static_cast<SegmentBase *>(Segment);
     return Segment;
   }
 
-  Segment *AppendNewSegment() {
+  Segment *AppendNewSegment() XRAY_NEVER_INSTRUMENT {
     auto S = NewSegment();
     if (S == nullptr)
       return nullptr;
     DCHECK_NE(Tail, &SentinelSegment);
     DCHECK_EQ(Tail->Next, &SentinelSegment);
     DCHECK_EQ(S->Prev, &SentinelSegment);
     DCHECK_EQ(S->Next, &SentinelSegment);
     Tail->Next = S;
     S->Prev = Tail;
     Tail = S;
     return static_cast<Segment *>(Tail);
   }
 
   // This Iterator models a BidirectionalIterator.
   template <class U> class Iterator {
     SegmentBase *S = &SentinelSegment;
     size_t Offset = 0;
     size_t Size = 0;
 
   public:
-    Iterator(SegmentBase *IS, size_t Off, size_t S)
-        : S(IS), Offset(Off), Size(S) {}
-    Iterator(const Iterator &) noexcept = default;
-    Iterator() noexcept = default;
-    Iterator(Iterator &&) noexcept = default;
-    Iterator &operator=(const Iterator &) = default;
-    Iterator &operator=(Iterator &&) = default;
-    ~Iterator() = default;
+    Iterator(SegmentBase *IS, size_t Off, size_t S) XRAY_NEVER_INSTRUMENT
+        : S(IS),
+          Offset(Off),
+          Size(S) {}
+    Iterator(const Iterator &) NOEXCEPT XRAY_NEVER_INSTRUMENT = default;
+    Iterator() NOEXCEPT XRAY_NEVER_INSTRUMENT = default;
+    Iterator(Iterator &&) NOEXCEPT XRAY_NEVER_INSTRUMENT = default;
+    Iterator &operator=(const Iterator &) XRAY_NEVER_INSTRUMENT = default;
+    Iterator &operator=(Iterator &&) XRAY_NEVER_INSTRUMENT = default;
+    ~Iterator() XRAY_NEVER_INSTRUMENT = default;
 
-    Iterator &operator++() {
+    Iterator &operator++() XRAY_NEVER_INSTRUMENT {
       if (++Offset % ElementsPerSegment || Offset == Size)
         return *this;
 
       // At this point, we know that Offset % N == 0, so we must advance the
       // segment pointer.
       DCHECK_EQ(Offset % ElementsPerSegment, 0);
       DCHECK_NE(Offset, Size);
       DCHECK_NE(S, &SentinelSegment);
       DCHECK_NE(S->Next, &SentinelSegment);
       S = S->Next;
       DCHECK_NE(S, &SentinelSegment);
       return *this;
     }
 
-    Iterator &operator--() {
+    Iterator &operator--() XRAY_NEVER_INSTRUMENT {
       DCHECK_NE(S, &SentinelSegment);
       DCHECK_GT(Offset, 0);
 
       auto PreviousOffset = Offset--;
       if (PreviousOffset != Size && PreviousOffset % ElementsPerSegment == 0) {
         DCHECK_NE(S->Prev, &SentinelSegment);
         S = S->Prev;
       }
 
       return *this;
     }
 
-    Iterator operator++(int) {
+    Iterator operator++(int) XRAY_NEVER_INSTRUMENT {
       Iterator Copy(*this);
       ++(*this);
       return Copy;
     }
 
-    Iterator operator--(int) {
+    Iterator operator--(int) XRAY_NEVER_INSTRUMENT {
       Iterator Copy(*this);
       --(*this);
       return Copy;
     }
 
     template <class V, class W>
-    friend bool operator==(const Iterator<V> &L, const Iterator<W> &R) {
+    friend bool operator==(const Iterator<V> &L,
+                           const Iterator<W> &R) XRAY_NEVER_INSTRUMENT {
       return L.S == R.S && L.Offset == R.Offset;
     }
 
     template <class V, class W>
-    friend bool operator!=(const Iterator<V> &L, const Iterator<W> &R) {
+    friend bool operator!=(const Iterator<V> &L,
+                           const Iterator<W> &R) XRAY_NEVER_INSTRUMENT {
       return !(L == R);
     }
 
-    U &operator*() const {
+    U &operator*() const XRAY_NEVER_INSTRUMENT {
       DCHECK_NE(S, &SentinelSegment);
       auto RelOff = Offset % ElementsPerSegment;
 
       // We need to compute the character-aligned pointer, offset from the
       // segment's Data location to get the element in the position of Offset.
       auto Base = static_cast<Segment *>(S)->Data;
       auto AlignedOffset = Base + (RelOff * AlignedElementStorageSize);
       return *reinterpret_cast<U *>(AlignedOffset);
     }
 
-    U *operator->() const { return &(**this); }
+    U *operator->() const XRAY_NEVER_INSTRUMENT { return &(**this); }
   };
 
 public:
-  explicit Array(AllocatorType &A) : Alloc(&A) {}
+  explicit Array(AllocatorType &A) XRAY_NEVER_INSTRUMENT : Alloc(&A) {}
 
   Array(const Array &) = delete;
   Array(Array &&O) NOEXCEPT : Alloc(O.Alloc),
                               Head(O.Head),
                               Tail(O.Tail),
                               Size(O.Size) {
     O.Head = &SentinelSegment;
     O.Tail = &SentinelSegment;
     O.Size = 0;
   }
 
-  bool empty() const { return Size == 0; }
+  bool empty() const XRAY_NEVER_INSTRUMENT { return Size == 0; }
 
-  AllocatorType &allocator() const {
+  AllocatorType &allocator() const XRAY_NEVER_INSTRUMENT {
     DCHECK_NE(Alloc, nullptr);
     return *Alloc;
   }
 
-  size_t size() const { return Size; }
+  size_t size() const XRAY_NEVER_INSTRUMENT { return Size; }
 
-  T *Append(const T &E) {
+  T *Append(const T &E) XRAY_NEVER_INSTRUMENT {
     if (UNLIKELY(Head == &SentinelSegment))
       if (InitHeadAndTail() == nullptr)
         return nullptr;
 
     auto Offset = Size % ElementsPerSegment;
     if (UNLIKELY(Size != 0 && Offset == 0))
       if (AppendNewSegment() == nullptr)
         return nullptr;
 
     auto Base = static_cast<Segment *>(Tail)->Data;
     auto AlignedOffset = Base + (Offset * AlignedElementStorageSize);
     auto Position = reinterpret_cast<T *>(AlignedOffset);
     *Position = E;
     ++Size;
     return Position;
   }
 
-  template <class... Args> T *AppendEmplace(Args &&... args) {
+  template <class... Args>
+  T *AppendEmplace(Args &&... args) XRAY_NEVER_INSTRUMENT {
     if (UNLIKELY(Head == &SentinelSegment))
       if (InitHeadAndTail() == nullptr)
         return nullptr;
 
     auto Offset = Size % ElementsPerSegment;
     auto *LatestSegment = Tail;
     if (UNLIKELY(Size != 0 && Offset == 0)) {
       LatestSegment = AppendNewSegment();
       if (LatestSegment == nullptr)
         return nullptr;
     }
 
     DCHECK_NE(Tail, &SentinelSegment);
     auto Base = static_cast<Segment *>(LatestSegment)->Data;
     auto AlignedOffset = Base + (Offset * AlignedElementStorageSize);
     auto Position = reinterpret_cast<T *>(AlignedOffset);
 
     // In-place construct at Position.
     new (Position) T{std::forward<Args>(args)...};
     ++Size;
     return reinterpret_cast<T *>(Position);
   }
 
-  T &operator[](size_t Offset) const {
+  T &operator[](size_t Offset) const XRAY_NEVER_INSTRUMENT {
     DCHECK_LE(Offset, Size);
     // We need to traverse the array enough times to find the element at Offset.
     auto S = Head;
     while (Offset >= ElementsPerSegment) {
       S = S->Next;
       Offset -= ElementsPerSegment;
       DCHECK_NE(S, &SentinelSegment);
     }
     auto Base = static_cast<Segment *>(S)->Data;
     auto AlignedOffset = Base + (Offset * AlignedElementStorageSize);
     auto Position = reinterpret_cast<T *>(AlignedOffset);
     return *reinterpret_cast<T *>(Position);
   }
 
-  T &front() const {
+  T &front() const XRAY_NEVER_INSTRUMENT {
     DCHECK_NE(Head, &SentinelSegment);
     DCHECK_NE(Size, 0u);
     return *begin();
   }
 
-  T &back() const {
+  T &back() const XRAY_NEVER_INSTRUMENT {
     DCHECK_NE(Tail, &SentinelSegment);
     DCHECK_NE(Size, 0u);
     auto It = end();
     --It;
     return *It;
   }
 
-  template <class Predicate> T *find_element(Predicate P) const {
+  template <class Predicate>
+  T *find_element(Predicate P) const XRAY_NEVER_INSTRUMENT {
     if (empty())
       return nullptr;
 
     auto E = end();
     for (auto I = begin(); I != E; ++I)
       if (P(*I))
         return &(*I);
 
     return nullptr;
   }
 
   /// Remove N Elements from the end. This leaves the blocks behind, and not
   /// require allocation of new blocks for new elements added after trimming.
-  void trim(size_t Elements) {
+  void trim(size_t Elements) XRAY_NEVER_INSTRUMENT {
     if (Elements == 0)
       return;
 
     DCHECK_LE(Elements, Size);
     DCHECK_GT(Size, 0);
     auto OldSize = Size;
     Size -= Elements;
 
for (auto SegmentsToTrim = (nearest_boundary(OldSize, ElementsPerSegment) -
       FreeSegment->Prev = &SentinelSegment;
       if (Freelist != &SentinelSegment)
         Freelist->Prev = FreeSegment;
       Freelist = FreeSegment;
     }
   }
 
   // Provide iterators.
-  Iterator<T> begin() const { return Iterator<T>(Head, 0, Size); }
-  Iterator<T> end() const { return Iterator<T>(Tail, Size, Size); }
-  Iterator<const T> cbegin() const { return Iterator<const T>(Head, 0, Size); }
-  Iterator<const T> cend() const { return Iterator<const T>(Tail, Size, Size); }
+  Iterator<T> begin() const XRAY_NEVER_INSTRUMENT {
+    return Iterator<T>(Head, 0, Size);
+  }
+  Iterator<T> end() const XRAY_NEVER_INSTRUMENT {
+    return Iterator<T>(Tail, Size, Size);
+  }
+  Iterator<const T> cbegin() const XRAY_NEVER_INSTRUMENT {
+    return Iterator<const T>(Head, 0, Size);
+  }
+  Iterator<const T> cend() const XRAY_NEVER_INSTRUMENT {
+    return Iterator<const T>(Tail, Size, Size);
+  }
 };
 
 // We need to have this storage definition out-of-line so that the compiler can
 // ensure that storage for the SentinelSegment is defined and has a single
 // address.
 template <class T>
 typename Array<T>::SegmentBase Array<T>::SentinelSegment{
     &Array<T>::SentinelSegment, &Array<T>::SentinelSegment};
 
 } // namespace __xray
 
 #endif // XRAY_SEGMENTED_ARRAY_H