[XRay] Refactor TSC related functions into a single header. NFC.

[timshen]

Summary

The implementation, however, is in different arch-specific files, unless it's emulated.

Tested on x86, but I didn't test on arm/aarch64. Will try to do so now.

[timshen]

Add inline and no-instrument to emulated TSC functions in the header.

[timshen]

Successfully compiled on aarch64.

compiler-rt/trunk/lib/xray/xray_AArch64.cc

 //===-- xray_AArch64.cc -----------------------------------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 // This file is a part of XRay, a dynamic runtime instrumentation system.
 //
 // Implementation of AArch64-specific routines (64-bit).
 //
 //===----------------------------------------------------------------------===//
 #include "sanitizer_common/sanitizer_common.h"
 #include "xray_defs.h"
-#include "xray_emulate_tsc.h"
 #include "xray_interface_internal.h"
 #include <atomic>
 #include <cassert>
 
 
 extern "C" void __clear_cache(void* start, void* end);
 
 namespace __xray {
 
-uint64_t cycleFrequency() XRAY_NEVER_INSTRUMENT {
-  // There is no instruction like RDTSCP in user mode on ARM.  ARM's CP15 does
-  //   not have a constant frequency like TSC on x86[_64]; it may go faster or
-  //   slower depending on CPU's turbo or power saving modes.  Furthermore, to
-  //   read from CP15 on ARM a kernel modification or a driver is needed.
-  //   We can not require this from users of compiler-rt.
-  // So on ARM we use clock_gettime(2) which gives the result in nanoseconds.
-  //   To get the measurements per second, we scale this by the number of
-  //   nanoseconds per second, pretending that the TSC frequency is 1GHz and
-  //   one TSC tick is 1 nanosecond.
-  return NanosecondsPerSecond;
-}
-
 // The machine codes for some instructions used in runtime patching.
 enum class PatchOpcodes : uint32_t {
   PO_StpX0X30SP_m16e = 0xA9BF7BE0, // STP X0, X30, [SP, #-16]!
   PO_LdrW0_12 = 0x18000060,        // LDR W0, #12
   PO_LdrX16_12 = 0x58000070,       // LDR X16, #12
   PO_BlrX16 = 0xD63F0200,          // BLR X16
   PO_LdpX0X30SP_16 = 0xA8C17BE0,   // LDP X0, X30, [SP], #16
   PO_B32 = 0x14000008              // B #32

compiler-rt/trunk/lib/xray/xray_arm.cc

 //===-- xray_arm.cc ---------------------------------------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 // This file is a part of XRay, a dynamic runtime instrumentation system.
 //
 // Implementation of ARM-specific routines (32-bit).
 //
 //===----------------------------------------------------------------------===//
 #include "sanitizer_common/sanitizer_common.h"
 #include "xray_defs.h"
-#include "xray_emulate_tsc.h"
 #include "xray_interface_internal.h"
 #include <atomic>
 #include <cassert>
 
 extern "C" void __clear_cache(void* start, void* end);
 
 namespace __xray {
 
-uint64_t cycleFrequency() XRAY_NEVER_INSTRUMENT {
-  // There is no instruction like RDTSCP in user mode on ARM.  ARM's CP15 does
-  //   not have a constant frequency like TSC on x86[_64]; it may go faster or
-  //   slower depending on CPU's turbo or power saving modes.  Furthermore, to
-  //   read from CP15 on ARM a kernel modification or a driver is needed.
-  //   We can not require this from users of compiler-rt.
-  // So on ARM we use clock_gettime(2) which gives the result in nanoseconds.
-  //   To get the measurements per second, we scale this by the number of
-  //   nanoseconds per second, pretending that the TSC frequency is 1GHz and
-  //   one TSC tick is 1 nanosecond.
-  return NanosecondsPerSecond;
-}
-
 // The machine codes for some instructions used in runtime patching.
 enum class PatchOpcodes : uint32_t {
   PO_PushR0Lr = 0xE92D4001, // PUSH {r0, lr}
   PO_BlxIp = 0xE12FFF3C,    // BLX ip
   PO_PopR0Lr = 0xE8BD4001,  // POP {r0, lr}
   PO_B20 = 0xEA000005       // B #20
 };
 

compiler-rt/trunk/lib/xray/xray_emulate_tsc.h

-//===-- xray_emulate_tsc.h --------------------------------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file is a part of XRay, a dynamic runtime instrumentation system.
-//
-//===----------------------------------------------------------------------===//
-#ifndef XRAY_EMULATE_TSC_H
-#define XRAY_EMULATE_TSC_H
-
-#include "sanitizer_common/sanitizer_common.h"
-#include "sanitizer_common/sanitizer_internal_defs.h"
-#include "xray_defs.h"
-#include <cerrno>
-#include <cstdint>
-#include <time.h>
-
-namespace __xray {
-
-static constexpr uint64_t NanosecondsPerSecond = 1000ULL * 1000 * 1000;
-
-ALWAYS_INLINE uint64_t readTSC(uint8_t &CPU) XRAY_NEVER_INSTRUMENT {
-  timespec TS;
-  int result = clock_gettime(CLOCK_REALTIME, &TS);
-  if (result != 0) {
-    Report("clock_gettime(2) returned %d, errno=%d.", result, int(errno));
-    TS.tv_sec = 0;
-    TS.tv_nsec = 0;
-  }
-  CPU = 0;
-  return TS.tv_sec * NanosecondsPerSecond + TS.tv_nsec;
-}
-
-bool probeRequiredCPUFeatures();
-
-} // namespace __xray
-
-#endif // XRAY_EMULATE_TSC_H

compiler-rt/trunk/lib/xray/xray_fdr_logging.cc

 #include <unordered_map>
 
 #include "sanitizer_common/sanitizer_common.h"
 #include "xray/xray_interface.h"
 #include "xray/xray_records.h"
 #include "xray_buffer_queue.h"
 #include "xray_defs.h"
 #include "xray_flags.h"
+#include "xray_tsc.h"
 #include "xray_utils.h"
 
-#if defined(__x86_64__)
-#include "xray_x86_64.h"
-#elif defined(__arm__) || defined(__aarch64__)
-#include "xray_emulate_tsc.h"
-#else
-#error "Unsupported CPU Architecture"
-#endif /* CPU architecture */
-
 namespace __xray {
 
 // Global BufferQueue.
 std::shared_ptr<BufferQueue> BQ;
 
 std::atomic<XRayLogInitStatus> LoggingStatus{
     XRayLogInitStatus::XRAY_LOG_UNINITIALIZED};
 
if (Fd == -1) {
     auto Result = XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING;
     LogFlushStatus.store(Result, std::memory_order_release);
     return Result;
   }
 
   XRayFileHeader Header;
   Header.Version = 1;
   Header.Type = FileTypes::FDR_LOG;
-  auto CPUFrequency = getCPUFrequency();
+  auto TSCFrequency = getTSCFrequency();
   Header.CycleFrequency =
-      CPUFrequency == -1 ? 0 : static_cast<uint64_t>(CPUFrequency);
+      TSCFrequency == -1 ? 0 : static_cast<uint64_t>(TSCFrequency);
   // FIXME: Actually check whether we have 'constant_tsc' and 'nonstop_tsc'
   // before setting the values in the header.
   Header.ConstantTSC = 1;
   Header.NonstopTSC = 1;
   clock_gettime(CLOCK_REALTIME, &Header.TS);
   retryingWriteAll(Fd, reinterpret_cast<char *>(&Header),
                    reinterpret_cast<char *>(&Header) + sizeof(Header));
   LocalBQ->apply([&](const BufferQueue::Buffer &B) {

compiler-rt/trunk/lib/xray/xray_inmemory_log.cc

 #include <fcntl.h>
 #include <mutex>
 #include <sys/stat.h>
 #include <sys/syscall.h>
 #include <sys/types.h>
 #include <thread>
 #include <unistd.h>
 
-#if defined(__x86_64__)
-#include "xray_x86_64.h"
-#elif defined(__arm__) || defined(__aarch64__)
-#include "xray_emulate_tsc.h"
-#else
-#error "Unsupported CPU Architecture"
-#endif /* Architecture-specific inline intrinsics */
-
 #include "sanitizer_common/sanitizer_libc.h"
 #include "xray/xray_records.h"
 #include "xray_defs.h"
 #include "xray_flags.h"
 #include "xray_interface_internal.h"
+#include "xray_tsc.h"
 #include "xray_utils.h"
 
 // __xray_InMemoryRawLog will use a thread-local aligned buffer capped to a
 // certain size (32kb by default) and use it as if it were a circular buffer for
 // events. We store simple fixed-sized entries in the log for external analysis.
 
 extern "C" {
 void __xray_InMemoryRawLog(int32_t FuncId,

 };
 
 } // namespace __xray
 
 using namespace __xray;
 
 static int __xray_OpenLogFile() XRAY_NEVER_INSTRUMENT {
   int F = getLogFD();
-  auto CPUFrequency = getCPUFrequency();
+  auto TSCFrequency = getTSCFrequency();
   if (F == -1)
     return -1;
   // Since we're here, we get to write the header. We set it up so that the
   // header will only be written once, at the start, and let the threads
   // logging do writes which just append.
   XRayFileHeader Header;
   Header.Version = 1;
   Header.Type = FileTypes::NAIVE_LOG;
   Header.CycleFrequency =
-      CPUFrequency == -1 ? 0 : static_cast<uint64_t>(CPUFrequency);
+      TSCFrequency == -1 ? 0 : static_cast<uint64_t>(TSCFrequency);
 
   // FIXME: Actually check whether we have 'constant_tsc' and 'nonstop_tsc'
   // before setting the values in the header.
   Header.ConstantTSC = 1;
   Header.NonstopTSC = 1;
   retryingWriteAll(F, reinterpret_cast<char *>(&Header),
                    reinterpret_cast<char *>(&Header) + sizeof(Header));
   return F;

compiler-rt/trunk/lib/xray/xray_interface_internal.h

 
 namespace __xray {
 
 struct XRaySledMap {
   const XRaySledEntry *Sleds;
   size_t Entries;
 };
 
-uint64_t cycleFrequency();
-
 bool patchFunctionEntry(bool Enable, uint32_t FuncId,
                         const XRaySledEntry &Sled);
 bool patchFunctionExit(bool Enable, uint32_t FuncId, const XRaySledEntry &Sled);
 bool patchFunctionTailExit(bool Enable, uint32_t FuncId,
                            const XRaySledEntry &Sled);
 
 } // namespace __xray
 

compiler-rt/trunk/lib/xray/xray_tsc.h

+//===-- xray_tsc.h ----------------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of XRay, a dynamic runtime instrumentation system.
+//
+//===----------------------------------------------------------------------===//
+#ifndef XRAY_EMULATE_TSC_H
+#define XRAY_EMULATE_TSC_H
+
+#if defined(__x86_64__)
+#include "xray_x86_64.inc"
+#elif defined(__arm__) || defined(__aarch64__)
+// Emulated TSC.
+// There is no instruction like RDTSCP in user mode on ARM. ARM's CP15 does
+//   not have a constant frequency like TSC on x86(_64), it may go faster
+//   or slower depending on CPU turbo or power saving mode. Furthermore,
+//   to read from CP15 on ARM a kernel modification or a driver is needed.
+//   We can not require this from users of compiler-rt.
+// So on ARM we use clock_gettime() which gives the result in nanoseconds.
+//   To get the measurements per second, we scale this by the number of
+//   nanoseconds per second, pretending that the TSC frequency is 1GHz and
+//   one TSC tick is 1 nanosecond.
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "xray_defs.h"
+#include <cerrno>
+#include <cstdint>
+#include <time.h>
+
+namespace __xray {
+
+static constexpr uint64_t NanosecondsPerSecond = 1000ULL * 1000 * 1000;
+
+inline bool probeRequiredCPUFeatures() XRAY_NEVER_INSTRUMENT { return true; }
+
+ALWAYS_INLINE uint64_t readTSC(uint8_t &CPU) XRAY_NEVER_INSTRUMENT {
+  timespec TS;
+  int result = clock_gettime(CLOCK_REALTIME, &TS);
+  if (result != 0) {
+    Report("clock_gettime(2) returned %d, errno=%d.", result, int(errno));
+    TS.tv_sec = 0;
+    TS.tv_nsec = 0;
+  }
+  CPU = 0;
+  return TS.tv_sec * NanosecondsPerSecond + TS.tv_nsec;
+}
+
+inline uint64_t getTSCFrequency() XRAY_NEVER_INSTRUMENT {
+  return NanosecondsPerSecond;
+}
+
+} // namespace __xray
+
+#else
+"Unsupported CPU Architecture"
+#endif // CPU architecture
+
+#endif // XRAY_EMULATE_TSC_H

compiler-rt/trunk/lib/xray/xray_utils.h

 
 // EINTR-safe read routine, providing a file descriptor and a character range.
 std::pair<ssize_t, bool> retryingReadSome(int Fd, char *Begin, char *End);
 
 // EINTR-safe open routine, uses flag-provided values for initialising a log
 // file.
 int getLogFD();
 
-// EINTR-safe read of CPU frquency for the current CPU.
-long long getCPUFrequency();
-
 } // namespace __xray
 
 #endif // XRAY_UTILS_H

compiler-rt/trunk/lib/xray/xray_utils.cc

 #include <errno.h>
 #include <fcntl.h>
 #include <iterator>
 #include <sys/types.h>
 #include <tuple>
 #include <unistd.h>
 #include <utility>
 
-#if defined(__x86_64__)
-#include "xray_x86_64.h"
-#elif defined(__arm__) || defined(__aarch64__)
-#include "xray_emulate_tsc.h"
-#else
-#error "Unsupported CPU Architecture"
-#endif /* CPU architecture */
-
 namespace __xray {
 
 void printToStdErr(const char *Buffer) XRAY_NEVER_INSTRUMENT {
   fprintf(stderr, "%s", Buffer);
 }
 
 void retryingWriteAll(int Fd, char *Begin, char *End) XRAY_NEVER_INSTRUMENT {
   if (Begin == End)
bool readValueFromFile(const char *Filename,
   bool Result = false;
   if (Line[0] != '\0' && (*End == '\n' || *End == '\0')) {
     *Value = Tmp;
     Result = true;
   }
   return Result;
 }
 
-long long getCPUFrequency() XRAY_NEVER_INSTRUMENT {
-  // Get the cycle frequency from SysFS on Linux.
-  long long CPUFrequency = -1;
-#if defined(__x86_64__)
-  if (readValueFromFile("/sys/devices/system/cpu/cpu0/tsc_freq_khz",
-                        &CPUFrequency)) {
-    CPUFrequency *= 1000;
-  } else if (readValueFromFile(
-                 "/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq",
-                 &CPUFrequency)) {
-    CPUFrequency *= 1000;
-  } else {
-    Report("Unable to determine CPU frequency for TSC accounting.\n");
-  }
-#elif defined(__arm__) || defined(__aarch64__)
-  // There is no instruction like RDTSCP in user mode on ARM. ARM's CP15 does
-  //   not have a constant frequency like TSC on x86(_64), it may go faster
-  //   or slower depending on CPU turbo or power saving mode. Furthermore,
-  //   to read from CP15 on ARM a kernel modification or a driver is needed.
-  //   We can not require this from users of compiler-rt.
-  // So on ARM we use clock_gettime() which gives the result in nanoseconds.
-  //   To get the measurements per second, we scale this by the number of
-  //   nanoseconds per second, pretending that the TSC frequency is 1GHz and
-  //   one TSC tick is 1 nanosecond.
-  CPUFrequency = NanosecondsPerSecond;
-#else
-#error "Unsupported CPU Architecture"
-#endif /* CPU architecture */
-  return CPUFrequency;
-}
-
 int getLogFD() XRAY_NEVER_INSTRUMENT {
   // FIXME: Figure out how to make this less stderr-dependent.
   SetPrintfAndReportCallback(printToStdErr);
   // Open a temporary file once for the log.
   static char TmpFilename[256] = {};
   static char TmpWildcardPattern[] = "XXXXXX";
   auto Argv = GetArgv();
   const char *Progname = Argv[0] == nullptr ? "(unknown)" : Argv[0];

compiler-rt/trunk/lib/xray/xray_x86_64.h

-//===-- xray_x86_64.h -------------------------------------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file is a part of XRay, a dynamic runtime instrumentation system.
-//
-//===----------------------------------------------------------------------===//
-#ifndef XRAY_X86_64_H
-#define XRAY_X86_64_H
-
-#include <cstdint>
-#include <x86intrin.h>
-
-#include "sanitizer_common/sanitizer_internal_defs.h"
-#include "xray_defs.h"
-
-namespace __xray {
-
-ALWAYS_INLINE uint64_t readTSC(uint8_t &CPU) XRAY_NEVER_INSTRUMENT {
-  unsigned LongCPU;
-  uint64_t TSC = __rdtscp(&LongCPU);
-  CPU = LongCPU;
-  return TSC;
-}
-
-bool probeRequiredCPUFeatures();
-
-} // namespace __xray
-
-#endif // XRAY_X86_64_H

compiler-rt/trunk/lib/xray/xray_x86_64.cc

static bool readValueFromFile(const char *Filename,
   bool Result = false;
   if (Line[0] != '\0' && (*End == '\n' || *End == '\0')) {
     *Value = Tmp;
     Result = true;
   }
   return Result;
 }
 
-uint64_t cycleFrequency() XRAY_NEVER_INSTRUMENT {
-  long long CPUFrequency = -1;
+uint64_t getTSCFrequency() XRAY_NEVER_INSTRUMENT {
+  long long TSCFrequency = -1;
   if (readValueFromFile("/sys/devices/system/cpu/cpu0/tsc_freq_khz",
-                        &CPUFrequency)) {
-    CPUFrequency *= 1000;
+                        &TSCFrequency)) {
+    TSCFrequency *= 1000;
   } else if (readValueFromFile(
                  "/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq",
-                 &CPUFrequency)) {
-    CPUFrequency *= 1000;
+                 &TSCFrequency)) {
+    TSCFrequency *= 1000;
   } else {
     Report("Unable to determine CPU frequency for TSC accounting.\n");
   }
-  return CPUFrequency == -1 ? 0 : static_cast<uint64_t>(CPUFrequency);
+  return TSCFrequency == -1 ? 0 : static_cast<uint64_t>(TSCFrequency);
 }
 
 static constexpr uint8_t CallOpCode = 0xe8;
 static constexpr uint16_t MovR10Seq = 0xba41;
 static constexpr uint16_t Jmp9Seq = 0x09eb;
 static constexpr uint8_t JmpOpCode = 0xe9;
 static constexpr uint8_t RetOpCode = 0xc3;
 

compiler-rt/trunk/lib/xray/xray_x86_64.inc

+//===-- xray_x86_64.inc -----------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of XRay, a dynamic runtime instrumentation system.
+//
+//===----------------------------------------------------------------------===//
+
+#include <cstdint>
+#include <x86intrin.h>
+
+#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "xray_defs.h"
+
+namespace __xray {
+
+ALWAYS_INLINE uint64_t readTSC(uint8_t &CPU) XRAY_NEVER_INSTRUMENT {
+  unsigned LongCPU;
+  uint64_t TSC = __rdtscp(&LongCPU);
+  CPU = LongCPU;
+  return TSC;
+}
+
+uint64_t getTSCFrequency();
+
+bool probeRequiredCPUFeatures();
+
+} // namespace __xray