[XRay][compiler-rt] Support string-based config for FDR mode

[dberris]

Summary

In this chage we add support for the string-based configuration
mechanism for configuring FDR mode.

We deprecate most of the xray_fdr_log_* flags that are set with the
XRAY_OPTIONS environment variable. Instead we make the FDR
implementation take defaults from the XRAY_FDR_OPTIONS environment
variable, and use the flags defined in xray_fdr_flags.{h,cc,inc} for
the options we support.

This change addresses http://llvm.org/PR36790.

Depends on D46173.

[dberris]

Rebase

[kpw]

Can you add some Reports, which could be guarded behind sanitizer verbose, for warning messages when users do deprecated things? This looks sensible, but a bit of nagging incentives people to change.

[dberris]

• fixup: Rebase and address comments by kpw@

[dberris]

Done, thanks!

lib/xray/CMakeLists.txt

 # Build for all components of the XRay runtime support library.
 
 # XRay runtime library implementation files.
 set(XRAY_SOURCES
     xray_init.cc
     xray_flags.cc
     xray_interface.cc
     xray_log_interface.cc
     xray_utils.cc)
 
 # Implementation files for all XRay modes.
 set(XRAY_FDR_MODE_SOURCES
+    xray_fdr_flags.cc
     xray_buffer_queue.cc
     xray_fdr_logging.cc)
 
 set(XRAY_BASIC_MODE_SOURCES
     xray_inmemory_log.cc)
 
 
 # Implementation files for all XRay architectures.

lib/xray/xray_fdr_flags.h

+//===-- xray_fdr_flags.h ---------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of XRay, a dynamic runtime instrumentation system.
+//
+// This file defines the flags for the flight-data-recorder mode implementation.
+//
+//===----------------------------------------------------------------------===//
+#ifndef XRAY_FDR_FLAGS_H
+#define XRAY_FDR_FLAGS_H
+
+#include "sanitizer_common/sanitizer_flag_parser.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
+
+namespace __xray {
+
+struct FDRFlags {
+#define XRAY_FLAG(Type, Name, DefaultValue, Description) Type Name;
+#include "xray_fdr_flags.inc"
+#undef XRAY_FLAG
+
+  void setDefaults();
+};
+
+extern FDRFlags xray_fdr_flags_dont_use_directly;
+extern void registerXRayFDRFlags(FlagParser *P, FDRFlags *F);
+const char *useCompilerDefinedFDRFlags();
+inline FDRFlags *fdrFlags() { return &xray_fdr_flags_dont_use_directly; }
+
+} // namespace __xray
+
+#endif // XRAY_FDR_FLAGS_H

lib/xray/xray_fdr_flags.cc

+//===-- xray_fdr_flags.cc ---------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of XRay, a dynamic runtime instrumentation system.
+//
+// XRay FDR flag parsing logic.
+//===----------------------------------------------------------------------===//
+
+#include "xray_fdr_flags.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_flag_parser.h"
+#include "sanitizer_common/sanitizer_libc.h"
+#include "xray_defs.h"
+
+using namespace __sanitizer;
+
+namespace __xray {
+
+FDRFlags xray_fdr_flags_dont_use_directly; // use via fdrFlags().
+
+void FDRFlags::setDefaults() XRAY_NEVER_INSTRUMENT {
+#define XRAY_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
+#include "xray_fdr_flags.inc"
+#undef XRAY_FLAG
+}
+
+void registerXRayFDRFlags(FlagParser *P, FDRFlags *F) XRAY_NEVER_INSTRUMENT {
+#define XRAY_FLAG(Type, Name, DefaultValue, Description)                       \
+  RegisterFlag(P, #Name, Description, &F->Name);
+#include "xray_fdr_flags.inc"
+#undef XRAY_FLAG
+}
+
+const char *useCompilerDefinedFDRFlags() XRAY_NEVER_INSTRUMENT {
+#ifdef XRAY_FDR_OPTIONS
+  return SANITIZER_STRINGIFY(XRAY_FDR_OPTIONS);
+#else
+  return "";
+#endif
+}
+
+} // namespace __xray

lib/xray/xray_fdr_flags.inc

+//===-- xray_fdr_flags.inc --------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// XRay FDR Mode runtime flags.
+//
+//===----------------------------------------------------------------------===//
+#ifndef XRAY_FLAG
+#error "Define XRAY_FLAG prior to including this file!"
+#endif
+
+// FDR (Flight Data Recorder) Mode logging options.
+XRAY_FLAG(int, func_duration_threshold_us, 5,
+          "FDR logging will try to skip functions that execute for fewer "
+          "microseconds than this threshold.")
+XRAY_FLAG(int, grace_period_ms, 100,
+          "FDR logging will wait this much time in milliseconds before "
+          "actually flushing the log; this gives a chance for threads to "
+          "notice that the log has been finalized and clean up.")
+XRAY_FLAG(int, buffer_size, 16384,
+          "Size of buffers in the circular buffer queue.")
+XRAY_FLAG(int, buffer_max, 100, "Maximum number of buffers in the queue.")

lib/xray/xray_fdr_logging.cc

 #include <unistd.h>
 
 #include "sanitizer_common/sanitizer_atomic.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "xray/xray_interface.h"
 #include "xray/xray_records.h"
 #include "xray_buffer_queue.h"
 #include "xray_defs.h"
+#include "xray_fdr_flags.h"
 #include "xray_fdr_logging_impl.h"
 #include "xray_flags.h"
 #include "xray_tsc.h"
 #include "xray_utils.h"
 
 namespace __xray {
 
 // Global BufferQueue.
XRayLogFlushStatus fdrLoggingFlush() XRAY_NEVER_INSTRUMENT {
   if (BQ == nullptr) {
     if (__sanitizer::Verbosity())
       Report("Cannot flush when global buffer queue is null.\n");
     return XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING;
   }
 
   // We wait a number of milliseconds to allow threads to see that we've
   // finalised before attempting to flush the log.
-  __sanitizer::SleepForMillis(flags()->xray_fdr_log_grace_period_ms);
+  __sanitizer::SleepForMillis(fdrFlags()->grace_period_ms);
 
   // We write out the file in the following format:
   //
   //   1) We write down the XRay file header with version 1, type FDR_LOG.
   //   2) Then we use the 'apply' member of the BufferQueue that's live, to
   //      ensure that at this point in time we write down the buffers that have
   //      been released (and marked "used") -- we dump the full buffer for now
   //      (fixed-sized) and let the tools reading the buffers deal with the data
   //      afterwards.
   //
+  // FIXME: Support the case for letting users handle the data through
+  // __xray_process_buffers(...) and provide an option to skip writing files.
   int Fd = -1;
   {
+    // FIXME: Remove this section of the code, when we remove the struct-based
+    // configuration API.
     __sanitizer::SpinMutexLock Guard(&FDROptionsMutex);
     Fd = FDROptions.Fd;
   }
   if (Fd == -1)
     Fd = getLogFD();
   if (Fd == -1) {
     auto Result = XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING;
     __sanitizer::atomic_store(&LogFlushStatus, Result,
XRayLogInitStatus fdrLoggingFinalize() XRAY_NEVER_INSTRUMENT {
   BQ->finalize();
 
   __sanitizer::atomic_store(&LoggingStatus,
                             XRayLogInitStatus::XRAY_LOG_FINALIZED,
                             __sanitizer::memory_order_release);
   return XRayLogInitStatus::XRAY_LOG_FINALIZED;
 }
 
-XRayLogInitStatus fdrLoggingReset() XRAY_NEVER_INSTRUMENT {
-  s32 CurrentStatus = XRayLogInitStatus::XRAY_LOG_FINALIZED;
-  if (__sanitizer::atomic_compare_exchange_strong(
-          &LoggingStatus, &CurrentStatus,
-          XRayLogInitStatus::XRAY_LOG_INITIALIZED,
-          __sanitizer::memory_order_release))
-    return static_cast<XRayLogInitStatus>(CurrentStatus);
-
-  // Release the in-memory buffer queue.
-  delete BQ;
-  BQ = nullptr;
-
-  // Spin until the flushing status is flushed.
-  s32 CurrentFlushingStatus = XRayLogFlushStatus::XRAY_LOG_FLUSHED;
-  while (__sanitizer::atomic_compare_exchange_weak(
-      &LogFlushStatus, &CurrentFlushingStatus,
-      XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING,
-      __sanitizer::memory_order_release)) {
-    if (CurrentFlushingStatus == XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING)
-      break;
-    CurrentFlushingStatus = XRayLogFlushStatus::XRAY_LOG_FLUSHED;
-  }
-
-  // At this point, we know that the status is flushed, and that we can assume
-  return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED;
-}
-
 struct TSCAndCPU {
   uint64_t TSC = 0;
   unsigned char CPU = 0;
 };
 
 static TSCAndCPU getTimestamp() XRAY_NEVER_INSTRUMENT {
   // We want to get the TSC as early as possible, so that we can check whether
   // we've seen this CPU before. We also do it before we load anything else, to
void fdrLoggingHandleTypedEvent(
   std::memcpy(TLD.RecordPtr, &TypedEvent, sizeof(TypedEvent));
 
   TLD.RecordPtr += sizeof(TypedEvent);
   std::memcpy(TLD.RecordPtr, Event, ReducedEventSize);
   incrementExtents(MetadataRecSize + EventSize);
   endBufferIfFull();
 }
 
-XRayLogInitStatus fdrLoggingInit(std::size_t BufferSize, std::size_t BufferMax,
+XRayLogInitStatus fdrLoggingInit(size_t BufferSize, size_t BufferMax,
                                  void *Options,
                                  size_t OptionsSize) XRAY_NEVER_INSTRUMENT {
-  if (OptionsSize != sizeof(FDRLoggingOptions)) {
-    if (__sanitizer::Verbosity())
-      Report("Cannot initialize FDR logging; wrong size for options: %d\n",
-             OptionsSize);
-    return static_cast<XRayLogInitStatus>(__sanitizer::atomic_load(
-        &LoggingStatus, __sanitizer::memory_order_acquire));
-  }
+  if (Options == nullptr)
+    return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED;
+
   s32 CurrentStatus = XRayLogInitStatus::XRAY_LOG_UNINITIALIZED;
   if (!__sanitizer::atomic_compare_exchange_strong(
           &LoggingStatus, &CurrentStatus,
           XRayLogInitStatus::XRAY_LOG_INITIALIZING,
           __sanitizer::memory_order_release)) {
     if (__sanitizer::Verbosity())
       Report("Cannot initialize already initialized implementation.\n");
     return static_cast<XRayLogInitStatus>(CurrentStatus);
   }
 
-  {
+  // Because of __xray_log_init_mode(...) which guarantees that this will be
+  // called with BufferSize == 0 and BufferMax == 0 we parse the configuration
+  // provided in the Options pointer as a string instead.
+  if (BufferSize == 0 && BufferMax == 0) {
+    if (__sanitizer::Verbosity())
+      Report("Initializing FDR mode with options: %s\n",
+             static_cast<const char *>(Options));
+
+    // TODO: Factor out the flags specific to the FDR mode implementation. For
+    // now, use the global/single definition of the flags, since the FDR mode
+    // flags are already defined there.
+    FlagParser FDRParser;
+    FDRFlags FDRFlags;
+    registerXRayFDRFlags(&FDRParser, &FDRFlags);
+    FDRFlags.setDefaults();
+
+    // Override first from the general XRAY_DEFAULT_OPTIONS compiler-provided
+    // options until we migrate everyone to use the XRAY_FDR_OPTIONS
+    // compiler-provided options.
+    FDRParser.ParseString(useCompilerDefinedFlags());
+    FDRParser.ParseString(useCompilerDefinedFDRFlags());
+    auto *EnvOpts = GetEnv("XRAY_FDR_OPTIONS");
+    if (EnvOpts == nullptr)
+      EnvOpts = "";
+    FDRParser.ParseString(EnvOpts);
+    FDRParser.ParseString(static_cast<const char *>(Options));
+    *fdrFlags() = FDRFlags;
+
+    BufferSize = FDRFlags.buffer_size;
+    BufferMax = FDRFlags.buffer_max;
+
+    // FIXME: Remove this when we fully remove the deprecated flags.
+    if (internal_strlen(EnvOpts) != 0) {
+      flags()->xray_fdr_log_func_duration_threshold_us =
+          FDRFlags.func_duration_threshold_us;
+      flags()->xray_fdr_log_grace_period_ms = FDRFlags.grace_period_ms;
+    }
+  } else if (OptionsSize != sizeof(FDRLoggingOptions)) {
+    // FIXME: This is deprecated, and should really be removed.
+    // At this point we use the flag parser specific to the FDR mode
+    // implementation.
+    if (__sanitizer::Verbosity())
+      Report("Cannot initialize FDR logging; wrong size for options: %d\n",
+             OptionsSize);
+    return static_cast<XRayLogInitStatus>(__sanitizer::atomic_load(
+        &LoggingStatus, __sanitizer::memory_order_acquire));
+  } else {
+    if (__sanitizer::Verbosity())
+      Report("XRay FDR: struct-based init is deprecated, please use "
+             "string-based configuration instead.\n");
     __sanitizer::SpinMutexLock Guard(&FDROptionsMutex);
     memcpy(&FDROptions, Options, OptionsSize);
   }
 
   bool Success = false;
 
   if (BQ != nullptr) {
     delete BQ;

lib/xray/xray_flags.h

 #define XRAY_FLAG(Type, Name, DefaultValue, Description) Type Name;
 #include "xray_flags.inc"
 #undef XRAY_FLAG
 
   void setDefaults();
 };
 
 extern Flags xray_flags_dont_use_directly;
+extern void registerXRayFlags(FlagParser *P, Flags *F);
+const char *useCompilerDefinedFlags();
 inline Flags *flags() { return &xray_flags_dont_use_directly; }
 
 void initializeFlags();
 
 } // namespace __xray
 
 #endif // XRAY_FLAGS_H

lib/xray/xray_flags.cc

 Flags xray_flags_dont_use_directly; // use via flags().
 
 void Flags::setDefaults() XRAY_NEVER_INSTRUMENT {
 #define XRAY_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
 #include "xray_flags.inc"
 #undef XRAY_FLAG
 }
 
-static void registerXRayFlags(FlagParser *P, Flags *F) XRAY_NEVER_INSTRUMENT {
+void registerXRayFlags(FlagParser *P, Flags *F) XRAY_NEVER_INSTRUMENT {
 #define XRAY_FLAG(Type, Name, DefaultValue, Description)                       \
   RegisterFlag(P, #Name, Description, &F->Name);
 #include "xray_flags.inc"
 #undef XRAY_FLAG
 }
 
 // This function, as defined with the help of a macro meant to be introduced at
 // build time of the XRay runtime, passes in a statically defined list of
 // options that control XRay. This means users/deployments can tweak the
 // defaults that override the hard-coded defaults in the xray_flags.inc at
 // compile-time using the XRAY_DEFAULT_OPTIONS macro.
-static const char *useCompilerDefinedFlags() XRAY_NEVER_INSTRUMENT {
+const char *useCompilerDefinedFlags() XRAY_NEVER_INSTRUMENT {
 #ifdef XRAY_DEFAULT_OPTIONS
   // Do the double-layered string conversion to prevent badly crafted strings
-// provided through the XRAY_DEFAULT_OPTIONS from causing compilation issues (or
-// changing the semantics of the implementation through the macro). This ensures
-// that we convert whatever XRAY_DEFAULT_OPTIONS is defined as a string literal.
+  // provided through the XRAY_DEFAULT_OPTIONS from causing compilation issues
+  // (or changing the semantics of the implementation through the macro). This
+  // ensures that we convert whatever XRAY_DEFAULT_OPTIONS is defined as a
+  // string literal.
   return SANITIZER_STRINGIFY(XRAY_DEFAULT_OPTIONS);
 #else
   return "";
 #endif
 }
 
 void initializeFlags() XRAY_NEVER_INSTRUMENT {
   SetCommonFlagsDefaults();

lib/xray/xray_flags.inc

XRAY_FLAG(int, xray_naive_log_max_stack_depth, 64,
           "any more and the recordings will be dropped.")
 XRAY_FLAG(int, xray_naive_log_thread_buffer_size, 1024,
           "The number of entries to keep on a per-thread buffer.")
 
 // FDR (Flight Data Recorder) Mode logging options.
 XRAY_FLAG(bool, xray_fdr_log, false,
           "DEPRECATED: Use xray_mode=xray-fdr instead.")
 XRAY_FLAG(int, xray_fdr_log_func_duration_threshold_us, 5,
-          "FDR logging will try to skip functions that execute for fewer "
-          "microseconds than this threshold.")
+          "DEPRECATED: use the environment variable XRAY_FDR_OPTIONS and set "
+          "func_duration_threshold_us instead.")
 XRAY_FLAG(int, xray_fdr_log_grace_period_us, 0,
-          "DEPRECATED: use xray_fdr_log_grace_period_ms instead.")
+          "DEPRECATED: use the environment variable XRAY_FDR_OPTIONS and set "
+          "grace_period_ms instead.")
 XRAY_FLAG(int, xray_fdr_log_grace_period_ms, 100,
-          "FDR logging will wait this much time in microseconds before "
-          "actually flushing the log; this gives a chance for threads to "
-          "notice that the log has been finalized and clean up.")
+          "DEPRECATED: use the environment variable XRAY_FDR_OPTIONS and set "
+          "grace_period_ms instead.")

test/xray/TestCases/Posix/fdr-mode.cc

 // RUN: %clangxx_xray -g -std=c++11 %s -o %t
 // RUN: rm fdr-logging-test-* || true
 // RUN: rm fdr-unwrite-test-* || true
-// RUN: XRAY_OPTIONS="patch_premain=false xray_naive_log=false xray_logfile_base=fdr-logging-test- xray_fdr_log=true verbosity=1 xray_fdr_log_func_duration_threshold_us=0" %run %t 2>&1 | FileCheck %s
-// RUN: XRAY_OPTIONS="patch_premain=false xray_naive_log=false xray_logfile_base=fdr-unwrite-test- xray_fdr_log=true verbosity=1 xray_fdr_log_func_duration_threshold_us=5000" %run %t 2>&1 | FileCheck %s
-// RUN: %llvm_xray convert --symbolize --output-format=yaml -instr_map=%t "`ls fdr-logging-test-* | head -1`" | FileCheck %s --check-prefix=TRACE
-// RUN: %llvm_xray convert --symbolize --output-format=yaml -instr_map=%t "`ls fdr-unwrite-test-* | head -1`" | FileCheck %s --check-prefix=UNWRITE
+// RUN: XRAY_OPTIONS="patch_premain=false xray_logfile_base=fdr-logging-test- \
+// RUN:     xray_mode=xray-fdr verbosity=1" \
+// RUN: XRAY_FDR_OPTIONS="func_duration_threshold_us=0" \
+// RUN:     %run %t 2>&1 | FileCheck %s
+// RUN: XRAY_OPTIONS="patch_premain=false \
+// RUN:     xray_logfile_base=fdr-unwrite-test- xray_mode=xray-fdr \
+// RUN:     verbosity=1" \
+// RUN: XRAY_FDR_OPTIONS="func_duration_threshold_us=5000" \
+// RUN:     %run %t 2>&1 | FileCheck %s
+// RUN: %llvm_xray convert --symbolize --output-format=yaml -instr_map=%t \
+// RUN:     "`ls fdr-logging-test-* | head -1`" \
+// RUN:     | FileCheck %s --check-prefix=TRACE
+// RUN: %llvm_xray convert --symbolize --output-format=yaml -instr_map=%t \
+// RUN:     "`ls fdr-unwrite-test-* | head -1`" \
+// RUN:     | FileCheck %s --check-prefix=UNWRITE
 // RUN: rm fdr-logging-test-*
 // RUN: rm fdr-unwrite-test-*
 // FIXME: Make llvm-xray work on non-x86_64 as well.
 // REQUIRES: x86_64-target-arch
 // REQUIRES: built-in-llvm-tree
 
 #include "xray/xray_log_interface.h"
 #include <cassert>
 #include <chrono>
 #include <cstdio>
 #include <iostream>
 #include <stdlib.h>
 #include <thread>
 #include <time.h>
 
-constexpr auto kBufferSize = 16384;
-constexpr auto kBufferMax = 10;
-
 thread_local uint64_t var = 0;
 [[clang::xray_always_instrument]] void __attribute__((noinline)) fC() { ++var; }
 
 [[clang::xray_always_instrument]] void __attribute__((noinline)) fB() { fC(); }
 
 [[clang::xray_always_instrument]] void __attribute__((noinline)) fA() { fB(); }
 
 [[clang::xray_always_instrument, clang::xray_log_args(1)]]
 void __attribute__((noinline)) fArg(int) { }
 
 int main(int argc, char *argv[]) {
   using namespace __xray;
-  FDRLoggingOptions Options;
   std::cout << "Logging before init." << std::endl;
   // CHECK: Logging before init.
-  auto status = __xray_log_init(kBufferSize, kBufferMax, &Options,
-                                sizeof(FDRLoggingOptions));
+  assert(__xray_log_select_mode("xray-fdr") ==
+         XRayLogRegisterStatus::XRAY_REGISTRATION_OK);
+  auto status =
+      __xray_log_init_mode("xray-fdr", "buffer_size=16384:buffer_max=10");
   assert(status == XRayLogInitStatus::XRAY_LOG_INITIALIZED);
   std::cout << "Init status " << status << std::endl;
   // CHECK: Init status {{.*}}
   std::cout << "Patching..." << std::endl;
   // CHECK: Patching...
   __xray_patch();
   fA();
   fC();