[XRay][compiler-rt] Always place the CPU record first for every buffer

[dberris]

Summary

In FDR Mode, when we set up a new buffer for a thread that's just
overflowed, we must place the CPU identifier with the TSC record as the
first record. This is so that we can reconstruct all the function
entry/exit with deltas rooted on a TSC record for the CPU at the
beginning of the buffer.

Without doing this, the tools are rejecting the log for cases when we've
overflown and have different buffers that don't have the CPU and TSC
records as the first entry in the buffers.

compiler-rt/trunk/lib/xray/xray_fdr_logging.cc

if (TSCSupported) {
     TSC = TS.tv_sec * __xray::NanosecondsPerSecond + TS.tv_nsec;
   }
   return std::make_tuple(TSC, CPU);
 }
 
 void fdrLoggingHandleArg0(int32_t FuncId,
                           XRayEntryType Entry) XRAY_NEVER_INSTRUMENT {
   auto TSC_CPU = getTimestamp();
-  __xray_fdr_internal::processFunctionHook(
-      FuncId, Entry, std::get<0>(TSC_CPU), std::get<1>(TSC_CPU), 0,
+  __xray_fdr_internal::processFunctionHook(FuncId, Entry, std::get<0>(TSC_CPU),
+                                           std::get<1>(TSC_CPU), 0,
                                            clock_gettime, *BQ);
 }
 
 void fdrLoggingHandleArg1(int32_t FuncId, XRayEntryType Entry,
                           uint64_t Arg) XRAY_NEVER_INSTRUMENT {
   auto TSC_CPU = getTimestamp();
-  __xray_fdr_internal::processFunctionHook(
-      FuncId, Entry, std::get<0>(TSC_CPU), std::get<1>(TSC_CPU), Arg,
+  __xray_fdr_internal::processFunctionHook(FuncId, Entry, std::get<0>(TSC_CPU),
+                                           std::get<1>(TSC_CPU), Arg,
                                            clock_gettime, *BQ);
 }
 
 void fdrLoggingHandleCustomEvent(void *Event,
                                  std::size_t EventSize) XRAY_NEVER_INSTRUMENT {
   using namespace __xray_fdr_internal;
   auto TSC_CPU = getTimestamp();
   auto &TSC = std::get<0>(TSC_CPU);
   auto &CPU = std::get<1>(TSC_CPU);
void fdrLoggingHandleCustomEvent(void *Event,
   auto &TLD = getThreadLocalData();
   if (!isLogInitializedAndReady(TLD.LocalBQ, TSC, CPU, clock_gettime))
     return;
 
   // Here we need to prepare the log to handle:
   //   - The metadata record we're going to write. (16 bytes)
   //   - The additional data we're going to write. Currently, that's the size of
   //   the event we're going to dump into the log as free-form bytes.
-  if (!prepareBuffer(clock_gettime, MetadataRecSize + EventSize)) {
+  if (!prepareBuffer(TSC, CPU, clock_gettime, MetadataRecSize + EventSize)) {
     TLD.LocalBQ = nullptr;
     return;
   }
 
   // Write the custom event metadata record, which consists of the following
   // information:
   //   - 8 bytes (64-bits) for the full TSC when the event started.
   //   - 4 bytes (32-bits) for the length of the data.

compiler-rt/trunk/lib/xray/xray_fdr_logging_impl.h

inline bool releaseThreadLocalBuffer(BufferQueue &BQArg) {
   if (EC != BufferQueue::ErrorCode::Ok) {
     Report("Failed to release buffer at %p; error=%s\n", TLD.Buffer.Buffer,
            BufferQueue::getErrorString(EC));
     return false;
   }
   return true;
 }
 
-inline bool prepareBuffer(int (*wall_clock_reader)(clockid_t,
+inline bool prepareBuffer(uint64_t TSC, unsigned char CPU,
+                          int (*wall_clock_reader)(clockid_t,
                                                    struct timespec *),
                           size_t MaxSize) XRAY_NEVER_INSTRUMENT {
   auto &TLD = getThreadLocalData();
   char *BufferStart = static_cast<char *>(TLD.Buffer.Buffer);
   if ((TLD.RecordPtr + MaxSize) >
       (BufferStart + TLD.Buffer.Size - MetadataRecSize)) {
     writeEOBMetadata();
     if (!releaseThreadLocalBuffer(*TLD.LocalBQ))
       return false;
     auto EC = TLD.LocalBQ->getBuffer(TLD.Buffer);
     if (EC != BufferQueue::ErrorCode::Ok) {
       Report("Failed to acquire a buffer; error=%s\n",
              BufferQueue::getErrorString(EC));
       return false;
     }
     setupNewBuffer(wall_clock_reader);
+
+    // Always write the CPU metadata as the first record in the buffer.
+    writeNewCPUIdMetadata(CPU, TSC);
   }
   return true;
 }
 
 inline bool isLogInitializedAndReady(
     std::shared_ptr<BufferQueue> &LBQ, uint64_t TSC, unsigned char CPU,
     int (*wall_clock_reader)(clockid_t,
                              struct timespec *)) XRAY_NEVER_INSTRUMENT {
if (EC != BufferQueue::ErrorCode::Ok) {
       if (LS != XRayLogInitStatus::XRAY_LOG_FINALIZING &&
           LS != XRayLogInitStatus::XRAY_LOG_FINALIZED)
         Report("Failed to acquire a buffer; error=%s\n",
                BufferQueue::getErrorString(EC));
       return false;
     }
 
     setupNewBuffer(wall_clock_reader);
+
+    // Always write the CPU metadata as the first record in the buffer.
+    writeNewCPUIdMetadata(CPU, TSC);
   }
 
   if (TLD.CurrentCPU == std::numeric_limits<uint16_t>::max()) {
     // This means this is the first CPU this thread has ever run on. We set
     // the current CPU and record this as the first TSC we've seen.
     TLD.CurrentCPU = CPU;
     writeNewCPUIdMetadata(CPU, TSC);
   }
inline void processFunctionHook(
   //   - An End-of-Buffer (EOB) MetadataRecord is 16 bytes.
   //
   // So the math we need to do is to determine whether writing 24 bytes past the
   // current pointer leaves us with enough bytes to write the EOB
   // MetadataRecord. If we don't have enough space after writing as much as 24
   // bytes in the end of the buffer, we need to write out the EOB, get a new
   // Buffer, set it up properly before doing any further writing.
   size_t MaxSize = FunctionRecSize + 2 * MetadataRecSize;
-  if (!prepareBuffer(wall_clock_reader, MaxSize)) {
+  if (!prepareBuffer(TSC, CPU, wall_clock_reader, MaxSize)) {
     TLD.LocalBQ = nullptr;
     return;
   }
 
   // By this point, we are now ready to write up to 40 bytes (explained above).
   assert((TLD.RecordPtr + MaxSize) - static_cast<char *>(TLD.Buffer.Buffer) >=
              static_cast<ptrdiff_t>(MetadataRecSize) &&
          "Misconfigured BufferQueue provided; Buffer size not large enough.");