[XRay] Define the library for XRay trace logs

[dberris]

Summary

In this change we move the definition of the log reading routines from
the tools directory in LLVM to {include/llvm,lib}/XRay. We improve the
documentation a little bit for the publicly accessible headers, and
adjust the top-matter. This also leads to some refactoring and cleanup
in the tooling code.

In particular, we do the following:

• Rename the class from LogReader to Trace, as it better represents

		the logical set of records as opposed to a log.

• Use file type detection instead of asking the user to say what

		format the input file is. This allows us to keep the interface
		simple and encapsulate the logic of loading the data appropriately.

In future changes we increase the API surface and write dedicated unit
tests for the XRay library.

Depends on D24376.

[dblaikie]

include/llvm/XRay/LogReader.h

39 ↗

(On Diff #83195)

Name seems slightly erroneous - rather than loading the log, this class seems to represent the log itself (you iterate over it as in the example, etc). Perhaps it should be called 'Log' instead? (or maybe there's a less ambiguous/likely colliding name) maybe with a named ctor (static function) called 'parse' instead of a ctor, for documentation?

53 ↗

(On Diff #83195)

Can the LogLoader use file magic to detect how to parse the file rather than having the user specify a LoaderFunction?

[dberris]

Use file type detection and rename LogReader to Trace.

[dberris]

Retitle the patch.

llvm/trunk/include/llvm/XRay/Trace.h

+//===- Trace.h - XRay Trace Abstraction -----------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines the XRay Trace class representing records in an XRay trace file.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_XRAY_TRACE_H
+#define LLVM_XRAY_TRACE_H
+
+#include <cstdint>
+#include <vector>
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/XRay/XRayRecord.h"
+
+namespace llvm {
+namespace xray {
+
+/// A Trace object represents the records that have been loaded from XRay
+/// log files generated by instrumented binaries. We encapsulate the logic of
+/// reading the traces in factory functions that populate the Trace object
+/// appropriately.
+///
+/// Trace objects provide an accessor to an XRayFileHeader which says more about
+/// details of the file from which the XRay trace was loaded from.
+///
+/// Usage:
+///
+///   if (auto TraceOrErr = loadTraceFile("xray-log.something.xray")) {
+///     auto& T = *TraceOrErr;
+///     // T.getFileHeader() will provide information from the trace header.
+///     for (const XRayRecord &R : T) {
+///       // ... do something with R here.
+///     }
+///   } else {
+///     // Handle the error here.
+///   }
+///
+class Trace {
+  XRayFileHeader FileHeader;
+  std::vector<XRayRecord> Records;
+
+  typedef std::vector<XRayRecord>::const_iterator citerator;
+
+  friend Expected<Trace> loadTraceFile(StringRef, bool);
+
+public:
+  /// Provides access to the loaded XRay trace file header.
+  const XRayFileHeader &getFileHeader() const { return FileHeader; }
+
+  citerator begin() const { return Records.begin(); }
+  citerator end() const { return Records.end(); }
+  size_t size() const { return Records.size(); }
+};
+
+/// This function will attempt to load XRay trace records from the provided
+/// |Filename|.
+Expected<Trace> loadTraceFile(StringRef Filename, bool Sort = false);
+
+} // namespace xray
+} // namespace llvm
+
+#endif // LLVM_XRAY_TRACE_H

llvm/trunk/include/llvm/XRay/XRayRecord.h

+//===- XRayRecord.h - XRay Trace Record -----------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file replicates the record definition for XRay log entries. This should
+// follow the evolution of the log record versions supported in the compiler-rt
+// xray project.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_XRAY_XRAY_RECORD_H
+#define LLVM_XRAY_XRAY_RECORD_H
+
+#include <cstdint>
+
+namespace llvm {
+namespace xray {
+
+/// XRay traces all have a header providing some top-matter information useful
+/// to help tools determine how to interpret the information available in the
+/// trace.
+struct XRayFileHeader {
+  /// Version of the XRay implementation that produced this file.
+  uint16_t Version = 0;
+
+  /// A numeric identifier for the type of file this is. Best used in
+  /// combination with Version.
+  uint16_t Type = 0;
+
+  /// Whether the CPU that produced the timestamp counters (TSC) move at a
+  /// constant rate.
+  bool ConstantTSC;
+
+  /// Whether the CPU that produced the timestamp counters (TSC) do not stop.
+  bool NonstopTSC;
+
+  /// The number of cycles per second for the CPU that produced the timestamp
+  /// counter (TSC) values. Useful for estimating the amount of time that
+  /// elapsed between two TSCs on some platforms.
+  uint64_t CycleFrequency = 0;
+};
+
+/// Determines the supported types of records that could be seen in XRay traces.
+/// This may or may not correspond to actual record types in the raw trace (as
+/// the loader implementation may synthesize this information in the process of
+/// of loading).
+enum class RecordTypes { ENTER, EXIT };
+
+struct XRayRecord {
+  /// The type of record.
+  uint16_t RecordType;
+
+  /// The CPU where the thread is running. We assume number of CPUs <= 256.
+  uint8_t CPU;
+
+  /// Identifies the type of record.
+  RecordTypes Type;
+
+  /// The function ID for the record.
+  int32_t FuncId;
+
+  /// Get the full 8 bytes of the TSC when we get the log record.
+  uint64_t TSC;
+
+  /// The thread ID for the currently running thread.
+  uint32_t TId;
+};
+
+} // namespace xray
+} // namespace llvm
+
+#endif // LLVM_XRAY_XRAY_RECORD_H

llvm/trunk/include/llvm/XRay/YAMLXRayRecord.h

+//===- YAMLXRayRecord.h - XRay Record YAML Support Definitions ------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Types and traits specialisations for YAML I/O of XRay log entries.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_XRAY_YAML_XRAY_RECORD_H
+#define LLVM_XRAY_YAML_XRAY_RECORD_H
+
+#include <type_traits>
+
+#include "llvm/Support/YAMLTraits.h"
+#include "llvm/XRay/XRayRecord.h"
+
+namespace llvm {
+namespace xray {
+
+struct YAMLXRayFileHeader {
+  uint16_t Version;
+  uint16_t Type;
+  bool ConstantTSC;
+  bool NonstopTSC;
+  uint64_t CycleFrequency;
+};
+
+struct YAMLXRayRecord {
+  uint16_t RecordType;
+  uint8_t CPU;
+  RecordTypes Type;
+  int32_t FuncId;
+  std::string Function;
+  uint64_t TSC;
+  uint32_t TId;
+};
+
+struct YAMLXRayTrace {
+  YAMLXRayFileHeader Header;
+  std::vector<YAMLXRayRecord> Records;
+};
+
+} // namespace xray
+
+namespace yaml {
+
+// YAML Traits
+// -----------
+template <> struct ScalarEnumerationTraits<xray::RecordTypes> {
+  static void enumeration(IO &IO, xray::RecordTypes &Type) {
+    IO.enumCase(Type, "function-enter", xray::RecordTypes::ENTER);
+    IO.enumCase(Type, "function-exit", xray::RecordTypes::EXIT);
+  }
+};
+
+template <> struct MappingTraits<xray::YAMLXRayFileHeader> {
+  static void mapping(IO &IO, xray::YAMLXRayFileHeader &Header) {
+    IO.mapRequired("version", Header.Version);
+    IO.mapRequired("type", Header.Type);
+    IO.mapRequired("constant-tsc", Header.ConstantTSC);
+    IO.mapRequired("nonstop-tsc", Header.NonstopTSC);
+    IO.mapRequired("cycle-frequency", Header.CycleFrequency);
+  }
+};
+
+template <> struct MappingTraits<xray::YAMLXRayRecord> {
+  static void mapping(IO &IO, xray::YAMLXRayRecord &Record) {
+    // FIXME: Make this type actually be descriptive
+    IO.mapRequired("type", Record.RecordType);
+    IO.mapRequired("func-id", Record.FuncId);
+    IO.mapOptional("function", Record.Function);
+    IO.mapRequired("cpu", Record.CPU);
+    IO.mapRequired("thread", Record.TId);
+    IO.mapRequired("kind", Record.Type);
+    IO.mapRequired("tsc", Record.TSC);
+  }
+
+  static constexpr bool flow = true;
+};
+
+template <> struct MappingTraits<xray::YAMLXRayTrace> {
+  static void mapping(IO &IO, xray::YAMLXRayTrace &Trace) {
+    // A trace file contains two parts, the header and the list of all the
+    // trace records.
+    IO.mapRequired("header", Trace.Header);
+    IO.mapRequired("records", Trace.Records);
+  }
+};
+
+} // namespace yaml
+} // namespace llvm
+
+LLVM_YAML_IS_SEQUENCE_VECTOR(xray::YAMLXRayRecord)
+
+#endif // LLVM_XRAY_YAML_XRAY_RECORD_H

llvm/trunk/lib/CMakeLists.txt

 add_subdirectory(ExecutionEngine)
 add_subdirectory(Target)
 add_subdirectory(AsmParser)
 add_subdirectory(LineEditor)
 add_subdirectory(ProfileData)
 add_subdirectory(Fuzzer)
 add_subdirectory(Passes)
 add_subdirectory(LibDriver)
+add_subdirectory(XRay)

llvm/trunk/lib/XRay/CMakeLists.txt

+add_llvm_library(LLVMXRay
+  Trace.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${LLVM_MAIN_INCLUDE_DIR}/llvm/ADT
+  ${LLVM_MAIN_INCLUDE_DIR}/llvm/XRay
+
+  DEPENDS
+  LLVMSupport
+
+  LINK_LIBS
+  LLVMSupport
+  )

llvm/trunk/lib/XRay/Trace.cpp

+//===- Trace.cpp - XRay Trace Loading implementation. ---------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// XRay log reader implementation.
+//
+//===----------------------------------------------------------------------===//
+#include "llvm/XRay/Trace.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/DataExtractor.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/XRay/YAMLXRayRecord.h"
+
+using namespace llvm;
+using namespace llvm::xray;
+using llvm::yaml::Input;
+
+using XRayRecordStorage =
+    std::aligned_storage<sizeof(XRayRecord), alignof(XRayRecord)>::type;
+
+Error NaiveLogLoader(StringRef Data, XRayFileHeader &FileHeader,
+                     std::vector<XRayRecord> &Records) {
+  // FIXME: Maybe deduce whether the data is little or big-endian using some
+  // magic bytes in the beginning of the file?
+
+  // First 32 bytes of the file will always be the header. We assume a certain
+  // format here:
+  //
+  //   (2)   uint16 : version
+  //   (2)   uint16 : type
+  //   (4)   uint32 : bitfield
+  //   (8)   uint64 : cycle frequency
+  //   (16)  -      : padding
+  //
+  if (Data.size() < 32)
+    return make_error<StringError>(
+        "Not enough bytes for an XRay log.",
+        std::make_error_code(std::errc::invalid_argument));
+
+  if (Data.size() - 32 == 0 || Data.size() % 32 != 0)
+    return make_error<StringError>(
+        "Invalid-sized XRay data.",
+        std::make_error_code(std::errc::invalid_argument));
+
+  DataExtractor HeaderExtractor(Data, true, 8);
+  uint32_t OffsetPtr = 0;
+  FileHeader.Version = HeaderExtractor.getU16(&OffsetPtr);
+  FileHeader.Type = HeaderExtractor.getU16(&OffsetPtr);
+  uint32_t Bitfield = HeaderExtractor.getU32(&OffsetPtr);
+  FileHeader.ConstantTSC = Bitfield & 1uL;
+  FileHeader.NonstopTSC = Bitfield & 1uL << 1;
+  FileHeader.CycleFrequency = HeaderExtractor.getU64(&OffsetPtr);
+
+  if (FileHeader.Version != 1)
+    return make_error<StringError>(
+        Twine("Unsupported XRay file version: ") + Twine(FileHeader.Version),
+        std::make_error_code(std::errc::invalid_argument));
+
+  // Each record after the header will be 32 bytes, in the following format:
+  //
+  //   (2)   uint16 : record type
+  //   (1)   uint8  : cpu id
+  //   (1)   uint8  : type
+  //   (4)   sint32 : function id
+  //   (8)   uint64 : tsc
+  //   (4)   uint32 : thread id
+  //   (12)  -      : padding
+  for (auto S = Data.drop_front(32); !S.empty(); S = S.drop_front(32)) {
+    DataExtractor RecordExtractor(S, true, 8);
+    uint32_t OffsetPtr = 0;
+    Records.emplace_back();
+    auto &Record = Records.back();
+    Record.RecordType = RecordExtractor.getU16(&OffsetPtr);
+    Record.CPU = RecordExtractor.getU8(&OffsetPtr);
+    auto Type = RecordExtractor.getU8(&OffsetPtr);
+    switch (Type) {
+    case 0:
+      Record.Type = RecordTypes::ENTER;
+      break;
+    case 1:
+      Record.Type = RecordTypes::EXIT;
+      break;
+    default:
+      return make_error<StringError>(
+          Twine("Unknown record type '") + Twine(int{Type}) + "'",
+          std::make_error_code(std::errc::executable_format_error));
+    }
+    Record.FuncId = RecordExtractor.getSigned(&OffsetPtr, sizeof(int32_t));
+    Record.TSC = RecordExtractor.getU64(&OffsetPtr);
+    Record.TId = RecordExtractor.getU32(&OffsetPtr);
+  }
+  return Error::success();
+}
+
+Error YAMLLogLoader(StringRef Data, XRayFileHeader &FileHeader,
+                    std::vector<XRayRecord> &Records) {
+
+  // Load the documents from the MappedFile.
+  YAMLXRayTrace Trace;
+  Input In(Data);
+  In >> Trace;
+  if (In.error())
+    return make_error<StringError>("Failed loading YAML Data.", In.error());
+
+  FileHeader.Version = Trace.Header.Version;
+  FileHeader.Type = Trace.Header.Type;
+  FileHeader.ConstantTSC = Trace.Header.ConstantTSC;
+  FileHeader.NonstopTSC = Trace.Header.NonstopTSC;
+  FileHeader.CycleFrequency = Trace.Header.CycleFrequency;
+
+  if (FileHeader.Version != 1)
+    return make_error<StringError>(
+        Twine("Unsupported XRay file version: ") + Twine(FileHeader.Version),
+        std::make_error_code(std::errc::invalid_argument));
+
+  Records.clear();
+  std::transform(Trace.Records.begin(), Trace.Records.end(),
+                 std::back_inserter(Records), [&](const YAMLXRayRecord &R) {
+                   return XRayRecord{R.RecordType, R.CPU, R.Type,
+                                     R.FuncId,     R.TSC, R.TId};
+                 });
+  return Error::success();
+}
+
+Expected<Trace> llvm::xray::loadTraceFile(StringRef Filename, bool Sort) {
+  int Fd;
+  if (auto EC = sys::fs::openFileForRead(Filename, Fd)) {
+    return make_error<StringError>(
+        Twine("Cannot read log from '") + Filename + "'", EC);
+  }
+
+  // Attempt to get the filesize.
+  uint64_t FileSize;
+  if (auto EC = sys::fs::file_size(Filename, FileSize)) {
+    return make_error<StringError>(
+        Twine("Cannot read log from '") + Filename + "'", EC);
+  }
+  if (FileSize < 4) {
+    return make_error<StringError>(
+        Twine("File '") + Filename + "' too small for XRay.",
+        std::make_error_code(std::errc::protocol_error));
+  }
+
+  // Attempt to mmap the file.
+  std::error_code EC;
+  sys::fs::mapped_file_region MappedFile(
+      Fd, sys::fs::mapped_file_region::mapmode::readonly, FileSize, 0, EC);
+  if (EC) {
+    return make_error<StringError>(
+        Twine("Cannot read log from '") + Filename + "'", EC);
+  }
+
+  // Attempt to detect the file type using file magic. We have a slight bias
+  // towards the binary format, and we do this by making sure that the first 4
+  // bytes of the binary file is some combination of the following byte
+  // patterns:
+  //
+  //   0x0001 0x0000 - version 1, "naive" format
+  //   0x0001 0x0001 - version 1, "flight data recorder" format
+  //
+  // YAML files dont' typically have those first four bytes as valid text so we
+  // try loading assuming YAML if we don't find these bytes.
+  //
+  // Only if we can't load either the binary or the YAML format will we yield an
+  // error.
+  StringRef Magic(MappedFile.data(), 4);
+  DataExtractor HeaderExtractor(Magic, true, 8);
+  uint32_t OffsetPtr = 0;
+  uint16_t Version = HeaderExtractor.getU16(&OffsetPtr);
+  uint16_t Type = HeaderExtractor.getU16(&OffsetPtr);
+
+  Trace T;
+  if (Version == 1 && (Type == 0 || Type == 1)) {
+    if (auto E = NaiveLogLoader(StringRef(MappedFile.data(), MappedFile.size()),
+                                T.FileHeader, T.Records))
+      return std::move(E);
+  } else {
+    if (auto E = YAMLLogLoader(StringRef(MappedFile.data(), MappedFile.size()),
+                               T.FileHeader, T.Records))
+      return std::move(E);
+  }
+
+  if (Sort)
+    std::sort(T.Records.begin(), T.Records.end(),
+              [&](const XRayRecord &L, const XRayRecord &R) {
+                return L.TSC < R.TSC;
+              });
+
+  return std::move(T);
+}

llvm/trunk/test/tools/llvm-xray/X86/convert-roundtrip.yaml

-#RUN: llvm-xray convert %s -i=yaml -f=raw -o %t && llvm-xray convert %t -f=yaml -o - | FileCheck %s
+#RUN: llvm-xray convert %s -f=raw -o %t && llvm-xray convert %t -f=yaml -o - | FileCheck %s
 ---
 header:
   version: 1
   type: 0
   constant-tsc: true
   nonstop-tsc: true
   cycle-frequency: 2601000000
 records:

llvm/trunk/tools/llvm-xray/CMakeLists.txt

 set(LLVM_LINK_COMPONENTS
   ${LLVM_TARGETS_TO_BUILD}
   DebugInfoDWARF
   Object
   Support
-  Symbolize)
+  Symbolize
+  XRay)
 
 set(LLVM_XRAY_TOOLS
   func-id-helper.cc
   xray-converter.cc
   xray-extract.cc
   xray-extract.cc
-  xray-log-reader.cc
   xray-registry.cc)
 
 add_llvm_tool(llvm-xray llvm-xray.cc ${LLVM_XRAY_TOOLS})

llvm/trunk/tools/llvm-xray/xray-converter.h

 // Defines the TraceConverter class for turning binary traces into
 // human-readable text and vice versa.
 //
 //===----------------------------------------------------------------------===//
 #ifndef LLVM_TOOLS_LLVM_XRAY_XRAY_CONVERTER_H
 #define LLVM_TOOLS_LLVM_XRAY_XRAY_CONVERTER_H
 
 #include "func-id-helper.h"
-#include "xray-log-reader.h"
-#include "xray-record.h"
+#include "llvm/XRay/XRayRecord.h"
+#include "llvm/XRay/Trace.h"
 
 namespace llvm {
 namespace xray {
 
 class TraceConverter {
   FuncIdConversionHelper &FuncIdHelper;
   bool Symbolize;
 
 public:
   TraceConverter(FuncIdConversionHelper &FuncIdHelper, bool Symbolize = false)
       : FuncIdHelper(FuncIdHelper), Symbolize(Symbolize) {}
 
-  void exportAsYAML(const LogReader &Records, raw_ostream &OS);
-  void exportAsRAWv1(const LogReader &Records, raw_ostream &OS);
+  void exportAsYAML(const Trace &Records, raw_ostream &OS);
+  void exportAsRAWv1(const Trace &Records, raw_ostream &OS);
 };
 
 } // namespace xray
 } // namespace llvm
 
 #endif // LLVM_TOOLS_LLVM_XRAY_XRAY_CONVERTER_H

llvm/trunk/tools/llvm-xray/xray-converter.cc

 //===- xray-converter.cc - XRay Trace Conversion --------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 // Implements the trace conversion functions.
 //
 //===----------------------------------------------------------------------===//
 #include "xray-converter.h"
 
 #include "xray-extract.h"
-#include "xray-record-yaml.h"
 #include "xray-registry.h"
 #include "llvm/DebugInfo/Symbolize/Symbolize.h"
 #include "llvm/Support/EndianStream.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/YAMLTraits.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/XRay/Trace.h"
+#include "llvm/XRay/YAMLXRayRecord.h"
 
 using namespace llvm;
 using namespace xray;
 
 // llvm-xray convert
 // ----------------------------------------------------------------------------
 static cl::SubCommand Convert("convert", "Trace Format Conversion");
 static cl::opt<std::string> ConvertInput(cl::Positional,
                                          cl::desc("<xray log file>"),
                                          cl::Required, cl::sub(Convert));
 enum class ConvertFormats { BINARY, YAML };
-static cl::opt<ConvertFormats> ConvertInputFormat(
-    "input-format", cl::desc("input format"),
-    cl::values(clEnumValN(ConvertFormats::BINARY, "raw",
-                          "input is in raw binary"),
-               clEnumValN(ConvertFormats::YAML, "yaml", "input is in yaml")),
-    cl::sub(Convert));
-static cl::alias ConvertInputFormat2("i", cl::aliasopt(ConvertInputFormat),
-                                     cl::desc("Alias for -input-format"),
-                                     cl::sub(Convert));
 static cl::opt<ConvertFormats> ConvertOutputFormat(
     "output-format", cl::desc("output format"),
     cl::values(clEnumValN(ConvertFormats::BINARY, "raw", "output in binary"),
                clEnumValN(ConvertFormats::YAML, "yaml", "output in yaml")),
     cl::sub(Convert));
 static cl::alias ConvertOutputFormat2("f", cl::aliasopt(ConvertOutputFormat),
                                       cl::desc("Alias for -output-format"),
                                       cl::sub(Convert));
cl::values(clEnumValN(InstrumentationMapExtractor::InputFormats::ELF, "elf",
                           "instrumentation map in an ELF header"),
                clEnumValN(InstrumentationMapExtractor::InputFormats::YAML,
                           "yaml", "instrumentation map in YAML")),
     cl::sub(Convert), cl::init(InstrumentationMapExtractor::InputFormats::ELF));
 static cl::alias InstrMapFormat2("t", cl::aliasopt(InstrMapFormat),
                                  cl::desc("Alias for -instr-map-format"),
                                  cl::sub(Convert));
 
-using llvm::yaml::MappingTraits;
-using llvm::yaml::ScalarEnumerationTraits;
 using llvm::yaml::IO;
 using llvm::yaml::Output;
 
-void TraceConverter::exportAsYAML(const LogReader &Records, raw_ostream &OS) {
+void TraceConverter::exportAsYAML(const Trace &Records, raw_ostream &OS) {
   YAMLXRayTrace Trace;
   const auto &FH = Records.getFileHeader();
   Trace.Header = {FH.Version, FH.Type, FH.ConstantTSC, FH.NonstopTSC,
                   FH.CycleFrequency};
   Trace.Records.reserve(Records.size());
   for (const auto &R : Records) {
     Trace.Records.push_back({R.RecordType, R.CPU, R.Type, R.FuncId,
                              Symbolize ? FuncIdHelper.SymbolOrNumber(R.FuncId)
                                        : std::to_string(R.FuncId),
                              R.TSC, R.TId});
   }
   Output Out(OS);
   Out << Trace;
 }
 
-void TraceConverter::exportAsRAWv1(const LogReader &Records, raw_ostream &OS) {
+void TraceConverter::exportAsRAWv1(const Trace &Records, raw_ostream &OS) {
   // First write out the file header, in the correct endian-appropriate format
   // (XRay assumes currently little endian).
   support::endian::Writer<support::endianness::little> Writer(OS);
   const auto &FH = Records.getFileHeader();
   Writer.write(FH.Version);
   Writer.write(FH.Type);
   uint32_t Bitfield{0};
   if (FH.ConstantTSC)
static CommandRegistration Unused(&Convert, []() -> Error {
 
   const auto &FunctionAddresses = Extractor.getFunctionAddresses();
   symbolize::LLVMSymbolizer::Options Opts(
       symbolize::FunctionNameKind::LinkageName, true, true, false, "");
   symbolize::LLVMSymbolizer Symbolizer(Opts);
   llvm::xray::FuncIdConversionHelper FuncIdHelper(ConvertInstrMap, Symbolizer,
                                                   FunctionAddresses);
   llvm::xray::TraceConverter TC(FuncIdHelper, ConvertSymbolize);
-  LogReader::LoaderFunction Loader;
-  switch (ConvertInputFormat) {
-  case ConvertFormats::BINARY:
-    Loader = NaiveLogLoader;
-    break;
-  case ConvertFormats::YAML:
-    Loader = YAMLLogLoader;
-    break;
-  }
-
-  LogReader Reader(ConvertInput, Err, ConvertSortInput, Loader);
-  if (Err)
-    return joinErrors(
-        make_error<StringError>(
-            Twine("Failed loading input file '") + ConvertInput + "'.",
-            std::make_error_code(std::errc::executable_format_error)),
-        std::move(Err));
-
   raw_fd_ostream OS(ConvertOutput, EC,
                     ConvertOutputFormat == ConvertFormats::BINARY
                         ? sys::fs::OpenFlags::F_None
                         : sys::fs::OpenFlags::F_Text);
   if (EC)
     return make_error<StringError>(
         Twine("Cannot open file '") + ConvertOutput + "' for writing.", EC);
 
+  if (auto TraceOrErr = loadTraceFile(ConvertInput, ConvertSortInput)) {
+    auto &T = *TraceOrErr;
     switch (ConvertOutputFormat) {
     case ConvertFormats::YAML:
-    TC.exportAsYAML(Reader, OS);
+      TC.exportAsYAML(T, OS);
       break;
     case ConvertFormats::BINARY:
-    TC.exportAsRAWv1(Reader, OS);
+      TC.exportAsRAWv1(T, OS);
       break;
     }
+  } else {
+    return joinErrors(
+        make_error<StringError>(
+            Twine("Failed loading input file '") + ConvertInput + "'.",
+            std::make_error_code(std::errc::protocol_error)),
+        TraceOrErr.takeError());
+  }
   return Error::success();
 });
 
 } // namespace xray
 } // namespace llvm

llvm/trunk/tools/llvm-xray/xray-log-reader.h

-//===- xray-log-reader.h - XRay Log Reader Interface ----------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Define the interface for an XRay log reader. Currently we only support one
-// version of the log (naive log) with fixed-sized records.
-//
-//===----------------------------------------------------------------------===//
-#ifndef LLVM_TOOLS_LLVM_XRAY_XRAY_LOG_READER_H
-#define LLVM_TOOLS_LLVM_XRAY_XRAY_LOG_READER_H
-
-#include <cstdint>
-#include <deque>
-#include <vector>
-
-#include "xray-record-yaml.h"
-#include "xray-record.h"
-#include "llvm/Support/Error.h"
-#include "llvm/Support/FileSystem.h"
-
-namespace llvm {
-namespace xray {
-
-class LogReader {
-  XRayFileHeader FileHeader;
-  std::vector<XRayRecord> Records;
-
-  typedef std::vector<XRayRecord>::const_iterator citerator;
-
-public:
-  typedef std::function<Error(StringRef, XRayFileHeader &,
-                              std::vector<XRayRecord> &)>
-      LoaderFunction;
-
-  LogReader(StringRef Filename, Error &Err, bool Sort, LoaderFunction Loader);
-
-  const XRayFileHeader &getFileHeader() const { return FileHeader; }
-
-  citerator begin() const { return Records.begin(); }
-  citerator end() const { return Records.end(); }
-  size_t size() const { return Records.size(); }
-};
-
-Error NaiveLogLoader(StringRef Data, XRayFileHeader &FileHeader,
-                     std::vector<XRayRecord> &Records);
-Error YAMLLogLoader(StringRef Data, XRayFileHeader &FileHeader,
-                    std::vector<XRayRecord> &Records);
-
-} // namespace xray
-} // namespace llvm
-
-#endif // LLVM_TOOLS_LLVM_XRAY_XRAY_LOG_READER_H

llvm/trunk/tools/llvm-xray/xray-log-reader.cc

-//===- xray-log-reader.cc - XRay Log Reader Implementation ----------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// XRay log reader implementation.
-//
-//===----------------------------------------------------------------------===//
-#include "xray-log-reader.h"
-#include "xray-record-yaml.h"
-#include "llvm/Support/DataExtractor.h"
-#include "llvm/Support/FileSystem.h"
-
-using namespace llvm;
-using namespace llvm::xray;
-using llvm::yaml::Input;
-
-LogReader::LogReader(
-    StringRef Filename, Error &Err, bool Sort,
-    std::function<Error(StringRef, XRayFileHeader &, std::vector<XRayRecord> &)>
-        Loader) {
-  ErrorAsOutParameter Guard(&Err);
-  int Fd;
-  if (auto EC = sys::fs::openFileForRead(Filename, Fd)) {
-    Err = make_error<StringError>(
-        Twine("Cannot read log from '") + Filename + "'", EC);
-    return;
-  }
-  uint64_t FileSize;
-  if (auto EC = sys::fs::file_size(Filename, FileSize)) {
-    Err = make_error<StringError>(
-        Twine("Cannot read log from '") + Filename + "'", EC);
-    return;
-  }
-
-  std::error_code EC;
-  sys::fs::mapped_file_region MappedFile(
-      Fd, sys::fs::mapped_file_region::mapmode::readonly, FileSize, 0, EC);
-  if (EC) {
-    Err = make_error<StringError>(
-        Twine("Cannot read log from '") + Filename + "'", EC);
-    return;
-  }
-
-  if (auto E = Loader(StringRef(MappedFile.data(), MappedFile.size()),
-                      FileHeader, Records)) {
-    Err = std::move(E);
-    return;
-  }
-
-  if (Sort)
-    std::sort(
-        Records.begin(), Records.end(),
-        [](const XRayRecord &L, const XRayRecord &R) { return L.TSC < R.TSC; });
-}
-
-Error llvm::xray::NaiveLogLoader(StringRef Data, XRayFileHeader &FileHeader,
-                                 std::vector<XRayRecord> &Records) {
-  // FIXME: Maybe deduce whether the data is little or big-endian using some
-  // magic bytes in the beginning of the file?
-
-  // First 32 bytes of the file will always be the header. We assume a certain
-  // format here:
-  //
-  //   (2)   uint16 : version
-  //   (2)   uint16 : type
-  //   (4)   uint32 : bitfield
-  //   (8)   uint64 : cycle frequency
-  //   (16)  -      : padding
-  //
-  if (Data.size() < 32)
-    return make_error<StringError>(
-        "Not enough bytes for an XRay log.",
-        std::make_error_code(std::errc::invalid_argument));
-
-  if (Data.size() - 32 == 0 || Data.size() % 32 != 0)
-    return make_error<StringError>(
-        "Invalid-sized XRay data.",
-        std::make_error_code(std::errc::invalid_argument));
-
-  DataExtractor HeaderExtractor(Data, true, 8);
-  uint32_t OffsetPtr = 0;
-  FileHeader.Version = HeaderExtractor.getU16(&OffsetPtr);
-  FileHeader.Type = HeaderExtractor.getU16(&OffsetPtr);
-  uint32_t Bitfield = HeaderExtractor.getU32(&OffsetPtr);
-  FileHeader.ConstantTSC = Bitfield & 1uL;
-  FileHeader.NonstopTSC = Bitfield & 1uL << 1;
-  FileHeader.CycleFrequency = HeaderExtractor.getU64(&OffsetPtr);
-
-  if (FileHeader.Version != 1)
-    return make_error<StringError>(
-        Twine("Unsupported XRay file version: ") + Twine(FileHeader.Version),
-        std::make_error_code(std::errc::invalid_argument));
-
-  // Each record after the header will be 32 bytes, in the following format:
-  //
-  //   (2)   uint16 : record type
-  //   (1)   uint8  : cpu id
-  //   (1)   uint8  : type
-  //   (4)   sint32 : function id
-  //   (8)   uint64 : tsc
-  //   (4)   uint32 : thread id
-  //   (12)  -      : padding
-  for (auto S = Data.drop_front(32); !S.empty(); S = S.drop_front(32)) {
-    DataExtractor RecordExtractor(S, true, 8);
-    uint32_t OffsetPtr = 0;
-    Records.emplace_back();
-    auto &Record = Records.back();
-    Record.RecordType = RecordExtractor.getU16(&OffsetPtr);
-    Record.CPU = RecordExtractor.getU8(&OffsetPtr);
-    auto Type = RecordExtractor.getU8(&OffsetPtr);
-    switch (Type) {
-    case 0:
-      Record.Type = RecordTypes::ENTER;
-      break;
-    case 1:
-      Record.Type = RecordTypes::EXIT;
-      break;
-    default:
-      return make_error<StringError>(
-          Twine("Unknown record type '") + Twine(int{Type}) + "'",
-          std::make_error_code(std::errc::executable_format_error));
-    }
-    Record.FuncId = RecordExtractor.getSigned(&OffsetPtr, sizeof(int32_t));
-    Record.TSC = RecordExtractor.getU64(&OffsetPtr);
-    Record.TId = RecordExtractor.getU32(&OffsetPtr);
-  }
-  return Error::success();
-}
-
-Error llvm::xray::YAMLLogLoader(StringRef Data, XRayFileHeader &FileHeader,
-                                std::vector<XRayRecord> &Records) {
-
-  // Load the documents from the MappedFile.
-  YAMLXRayTrace Trace;
-  Input In(Data);
-  In >> Trace;
-  if (In.error())
-    return make_error<StringError>("Failed loading YAML Data.", In.error());
-
-  FileHeader.Version = Trace.Header.Version;
-  FileHeader.Type = Trace.Header.Type;
-  FileHeader.ConstantTSC = Trace.Header.ConstantTSC;
-  FileHeader.NonstopTSC = Trace.Header.NonstopTSC;
-  FileHeader.CycleFrequency = Trace.Header.CycleFrequency;
-
-  if (FileHeader.Version != 1)
-    return make_error<StringError>(
-        Twine("Unsupported XRay file version: ") + Twine(FileHeader.Version),
-        std::make_error_code(std::errc::invalid_argument));
-
-  Records.clear();
-  std::transform(Trace.Records.begin(), Trace.Records.end(),
-                 std::back_inserter(Records), [&](const YAMLXRayRecord &R) {
-                   return XRayRecord{R.RecordType, R.CPU, R.Type,
-                                     R.FuncId,     R.TSC, R.TId};
-                 });
-  return Error::success();
-}

llvm/trunk/tools/llvm-xray/xray-record-yaml.h

-//===- xray-record-yaml.h - XRay Record YAML Support Definitions ----------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Types and traits specialisations for YAML I/O of XRay log entries.
-//
-//===----------------------------------------------------------------------===//
-#ifndef LLVM_TOOLS_LLVM_XRAY_XRAY_RECORD_YAML_H
-#define LLVM_TOOLS_LLVM_XRAY_XRAY_RECORD_YAML_H
-
-#include <type_traits>
-
-#include "xray-record.h"
-#include "llvm/Support/YAMLTraits.h"
-
-namespace llvm {
-namespace xray {
-
-struct YAMLXRayFileHeader {
-  uint16_t Version;
-  uint16_t Type;
-  bool ConstantTSC;
-  bool NonstopTSC;
-  uint64_t CycleFrequency;
-};
-
-struct YAMLXRayRecord {
-  uint16_t RecordType;
-  uint8_t CPU;
-  RecordTypes Type;
-  int32_t FuncId;
-  std::string Function;
-  uint64_t TSC;
-  uint32_t TId;
-};
-
-struct YAMLXRayTrace {
-  YAMLXRayFileHeader Header;
-  std::vector<YAMLXRayRecord> Records;
-};
-
-using XRayRecordStorage =
-    std::aligned_storage<sizeof(XRayRecord), alignof(XRayRecord)>::type;
-
-} // namespace xray
-
-namespace yaml {
-
-// YAML Traits
-// -----------
-template <> struct ScalarEnumerationTraits<xray::RecordTypes> {
-  static void enumeration(IO &IO, xray::RecordTypes &Type) {
-    IO.enumCase(Type, "function-enter", xray::RecordTypes::ENTER);
-    IO.enumCase(Type, "function-exit", xray::RecordTypes::EXIT);
-  }
-};
-
-template <> struct MappingTraits<xray::YAMLXRayFileHeader> {
-  static void mapping(IO &IO, xray::YAMLXRayFileHeader &Header) {
-    IO.mapRequired("version", Header.Version);
-    IO.mapRequired("type", Header.Type);
-    IO.mapRequired("constant-tsc", Header.ConstantTSC);
-    IO.mapRequired("nonstop-tsc", Header.NonstopTSC);
-    IO.mapRequired("cycle-frequency", Header.CycleFrequency);
-  }
-};
-
-template <> struct MappingTraits<xray::YAMLXRayRecord> {
-  static void mapping(IO &IO, xray::YAMLXRayRecord &Record) {
-    // FIXME: Make this type actually be descriptive
-    IO.mapRequired("type", Record.RecordType);
-    IO.mapRequired("func-id", Record.FuncId);
-    IO.mapOptional("function", Record.Function);
-    IO.mapRequired("cpu", Record.CPU);
-    IO.mapRequired("thread", Record.TId);
-    IO.mapRequired("kind", Record.Type);
-    IO.mapRequired("tsc", Record.TSC);
-  }
-
-  static constexpr bool flow = true;
-};
-
-template <> struct MappingTraits<xray::YAMLXRayTrace> {
-  static void mapping(IO &IO, xray::YAMLXRayTrace &Trace) {
-    // A trace file contains two parts, the header and the list of all the
-    // trace records.
-    IO.mapRequired("header", Trace.Header);
-    IO.mapRequired("records", Trace.Records);
-  }
-};
-
-} // namespace yaml
-} // namespace llvm
-
-LLVM_YAML_IS_SEQUENCE_VECTOR(xray::YAMLXRayRecord) 
-
-#endif // LLVM_TOOLS_LLVM_XRAY_XRAY_RECORD_YAML_H

llvm/trunk/tools/llvm-xray/xray-record.h

-//===- xray-record.h - XRay Trace Record ----------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file replicates the record definition for XRay log entries. This should
-// follow the evolution of the log record versions supported in the compiler-rt
-// xray project.
-//
-//===----------------------------------------------------------------------===//
-#ifndef LLVM_TOOLS_LLVM_XRAY_XRAY_RECORD_H
-#define LLVM_TOOLS_LLVM_XRAY_XRAY_RECORD_H
-
-#include <cstdint>
-
-namespace llvm {
-namespace xray {
-
-struct XRayFileHeader {
-  uint16_t Version = 0;
-  uint16_t Type = 0;
-  bool ConstantTSC;
-  bool NonstopTSC;
-  uint64_t CycleFrequency = 0;
-};
-
-enum class RecordTypes { ENTER, EXIT };
-
-struct XRayRecord {
-  uint16_t RecordType;
-
-  // The CPU where the thread is running. We assume number of CPUs <= 256.
-  uint8_t CPU;
-
-  // Identifies the type of record.
-  RecordTypes Type;
-
-  // The function ID for the record.
-  int32_t FuncId;
-
-  // Get the full 8 bytes of the TSC when we get the log record.
-  uint64_t TSC;
-
-  // The thread ID for the currently running thread.
-  uint32_t TId;
-};
-
-} // namespace xray
-} // namespace llvm
-
-#endif // LLVM_TOOLS_LLVM_XRAY_XRAY_RECORD_H