[XRay] Add the `llvm-xray fdr-dump` implementation

[dberris]

Summary

In this change, we implement a BlockPrinter which orders records in a
Block that's been indexed by the BlockIndexer. This is used in the
llvm-xray fdr-dump tool which ties together the various types and
utilities we've been working on, to allow for inspection of XRay FDR
mode traces both with and without verification.

This change is the final step of the refactoring of D50441.

[dberris]

Update description.

[mboerger]

LGTM

llvm/include/llvm/XRay/BlockPrinter.h

+//===- BlockPrinter.h - FDR Block Pretty Printer -------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// An implementation of the RecordVisitor which formats a block of records for
+// easier human consumption.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_INCLUDE_LLVM_XRAY_BLOCKPRINTER_H_
+#define LLVM_INCLUDE_LLVM_XRAY_BLOCKPRINTER_H_
+
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/XRay/FDRRecords.h"
+#include "llvm/XRay/RecordPrinter.h"
+
+namespace llvm {
+namespace xray {
+
+class BlockPrinter : public RecordVisitor {
+  enum class State {
+    Start,
+    Preamble,
+    Metadata,
+    Function,
+    Arg,
+    CustomEvent,
+    End,
+  };
+
+  raw_ostream &OS;
+  RecordPrinter &RP;
+  State CurrentState = State::Start;
+
+public:
+  explicit BlockPrinter(raw_ostream &O, RecordPrinter &P)
+      : RecordVisitor(), OS(O), RP(P) {}
+
+  Error visit(BufferExtents &) override;
+  Error visit(WallclockRecord &) override;
+  Error visit(NewCPUIDRecord &) override;
+  Error visit(TSCWrapRecord &) override;
+  Error visit(CustomEventRecord &) override;
+  Error visit(CallArgRecord &) override;
+  Error visit(PIDRecord &) override;
+  Error visit(NewBufferRecord &) override;
+  Error visit(EndBufferRecord &) override;
+  Error visit(FunctionRecord &) override;
+
+  void reset() { CurrentState = State::Start; }
+};
+
+} // namespace xray
+} // namespace llvm
+
+#endif // LLVM_INCLUDE_LLVM_XRAY_BLOCKPRINTER_H_

llvm/lib/XRay/BlockPrinter.cpp

+//===- BlockPrinter.cpp - FDR Block Pretty Printer Implementation --------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+#include "llvm/XRay/BlockPrinter.h"
+
+namespace llvm {
+namespace xray {
+
+Error BlockPrinter::visit(BufferExtents &R) {
+  OS << "\n[New Block]\n";
+  CurrentState = State::Preamble;
+  return RP.visit(R);
+}
+
+// Preamble printing.
+Error BlockPrinter::visit(NewBufferRecord &R) {
+  if (CurrentState == State::Start)
+    OS << "\n[New Block]\n";
+
+  OS << "Preamble: \n";
+  CurrentState = State::Preamble;
+  return RP.visit(R);
+}
+
+Error BlockPrinter::visit(WallclockRecord &R) {
+  CurrentState = State::Preamble;
+  return RP.visit(R);
+}
+
+Error BlockPrinter::visit(PIDRecord &R) {
+  CurrentState = State::Preamble;
+  return RP.visit(R);
+}
+
+// Metadata printing.
+Error BlockPrinter::visit(NewCPUIDRecord &R) {
+  if (CurrentState == State::Preamble)
+    OS << "\nBody:\n";
+  if (CurrentState == State::Function)
+    OS << "\nMetadata: ";
+  CurrentState = State::Metadata;
+  OS << " ";
+  auto E = RP.visit(R);
+  return E;
+}
+
+Error BlockPrinter::visit(TSCWrapRecord &R) {
+  if (CurrentState == State::Function)
+    OS << "\nMetadata:";
+  CurrentState = State::Metadata;
+  OS << " ";
+  auto E = RP.visit(R);
+  return E;
+}
+
+// Custom events will be rendered like "function" events.
+Error BlockPrinter::visit(CustomEventRecord &R) {
+  if (CurrentState == State::Metadata)
+    OS << "\n";
+  CurrentState = State::CustomEvent;
+  OS << "*  ";
+  auto E = RP.visit(R);
+  return E;
+}
+
+// Function call printing.
+Error BlockPrinter::visit(FunctionRecord &R) {
+  if (CurrentState == State::Metadata)
+    OS << "\n";
+  CurrentState = State::Function;
+  OS << "-  ";
+  auto E = RP.visit(R);
+  return E;
+}
+
+Error BlockPrinter::visit(CallArgRecord &R) {
+  CurrentState = State::Arg;
+  OS << " : ";
+  auto E = RP.visit(R);
+  return E;
+}
+
+Error BlockPrinter::visit(EndBufferRecord &R) {
+    CurrentState = State::End;
+    OS << " *** ";
+    auto E = RP.visit(R);
+    return E;
+}
+
+} // namespace xray
+} // namespace llvm

llvm/lib/XRay/CMakeLists.txt

 add_llvm_library(LLVMXRay
   BlockIndexer.cpp
+  BlockPrinter.cpp
   BlockVerifier.cpp
   FDRRecordProducer.cpp
   FDRRecords.cpp
   FDRTraceWriter.cpp
   FileHeaderReader.cpp
   InstrumentationMap.cpp
   LogBuilderConsumer.cpp
   Profile.cpp

llvm/test/tools/llvm-xray/X86/fdr-dump-arg1-version-3.txt

+; RUN: llvm-xray fdr-dump -verify %S/Inputs/fdr-log-arg1-version-3.xray \
+; RUN:     | FileCheck %s
+
+; CHECK:      [New Block]
+; CHECK-NEXT: Preamble:
+; CHECK-NEXT: <Thread ID: 2631>
+; CHECK-NEXT: <Wall Time: seconds = 599605.032403>
+; CHECK-NEXT: <PID: 2631>
+; CHECK-EMPTY:
+; CHECK-NEXT: Body:
+; CHECK-NEXT:  <CPU ID: 6>
+; CHECK-NEXT:  <CPU ID: 6>
+; CHECK-NEXT:  <TSC Wrap: base = 2034042117104344>
+; CHECK-EMPTY:
+; CHECK-NEXT: -  <Function Enter: #3 delta = +3>
+; CHECK-NEXT: -  <Function Exit: #3 delta = +3>
+; CHECK-NEXT: -  <Function Enter: #2 delta = +2>
+; CHECK-NEXT: -  <Function Exit: #2 delta = +2>
+; CHECK-EMPTY:
+; CHECK-NEXT: Metadata: <TSC Wrap: base = 2034049739853430>
+; CHECK-EMPTY:
+; CHECK-NEXT: -  <Function Enter: #1 delta = +1>
+; CHECK-NEXT:  : <Call Argument: data = 67 (hex = 0x43)>
+; CHECK-NEXT: -  <Function Exit: #1 delta = +1>
+

llvm/test/tools/llvm-xray/X86/fdr-dump-arg1.txt

+; RUN: llvm-xray fdr-dump -verify %S/Inputs/fdr-log-arg1.xray | FileCheck %s
+
+; CHECK:      [New Block]
+; CHECK-NEXT: Preamble:
+; CHECK-NEXT: <Thread ID: 14648>
+; CHECK-NEXT: <Wall Time: seconds = 1452786.250689>
+; CHECK-EMPTY:
+; CHECK-NEXT: Body:
+; CHECK-NEXT:  <CPU ID: 49>
+; CHECK-NEXT:  <TSC Wrap: base = 18828908666540172>
+; CHECK-EMPTY:
+; CHECK-NEXT: -  <Function Enter: #1 delta = +1>
+; CHECK-NEXT:  : <Call Argument: data = 1 (hex = 0x1)>
+; CHECK-NEXT: -  <Function Exit: #1 delta = +1>
+; CHECK-NEXT:  *** <End of Buffer>
+

llvm/tools/llvm-xray/CMakeLists.txt

 set(LLVM_LINK_COMPONENTS
   ${LLVM_TARGETS_TO_BUILD}
   DebugInfoDWARF
   Object
   Support
   Symbolize
   XRay
   )
 
 add_llvm_tool(llvm-xray
   func-id-helper.cpp
   llvm-xray.cpp
   xray-account.cpp
   xray-color-helper.cpp
   xray-converter.cpp
   xray-extract.cpp
+  xray-fdr-dump.cpp
   xray-graph-diff.cpp
   xray-graph.cpp
   xray-registry.cpp
   xray-stacks.cpp
   )

llvm/tools/llvm-xray/xray-fdr-dump.cpp

+//===- xray-fdr-dump.cpp: XRay FDR Trace Dump Tool ------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Implements the FDR trace dumping tool, using the libraries for handling FDR
+// mode traces specifically.
+//
+//===----------------------------------------------------------------------===//
+#include "xray-registry.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/XRay/BlockIndexer.h"
+#include "llvm/XRay/BlockPrinter.h"
+#include "llvm/XRay/BlockVerifier.h"
+#include "llvm/XRay/FDRRecordConsumer.h"
+#include "llvm/XRay/FDRRecordProducer.h"
+#include "llvm/XRay/FDRRecords.h"
+#include "llvm/XRay/FileHeaderReader.h"
+#include "llvm/XRay/RecordPrinter.h"
+
+using namespace llvm;
+using namespace xray;
+
+static cl::SubCommand Dump("fdr-dump", "FDR Trace Dump");
+static cl::opt<std::string> DumpInput(cl::Positional,
+                                      cl::desc("<xray fdr mode log>"),
+                                      cl::Required, cl::sub(Dump));
+static cl::opt<bool> DumpVerify("verify",
+                                cl::desc("verify structure of the log"),
+                                cl::init(false), cl::sub(Dump));
+
+static CommandRegistration Unused(&Dump, []() -> Error {
+  // Open the file provided.
+  int Fd;
+  if (auto EC = sys::fs::openFileForRead(DumpInput, Fd))
+    return createStringError(EC, "Cannot open file '%s' for read.",
+                             DumpInput.c_str());
+
+  uint64_t FileSize;
+  if (auto EC = sys::fs::file_size(DumpInput, FileSize))
+    return createStringError(EC, "Failed to get file size for '%s'.",
+                             DumpInput.c_str());
+
+  std::error_code EC;
+  sys::fs::mapped_file_region MappedFile(
+      Fd, sys::fs::mapped_file_region::mapmode::readonly, FileSize, 0, EC);
+
+  DataExtractor DE(StringRef(MappedFile.data(), MappedFile.size()), true, 8);
+  uint32_t OffsetPtr = 0;
+
+  auto FileHeaderOrError = readBinaryFormatHeader(DE, OffsetPtr);
+  if (!FileHeaderOrError)
+    return FileHeaderOrError.takeError();
+  auto &H = FileHeaderOrError.get();
+
+  FileBasedRecordProducer P(H, DE, OffsetPtr);
+
+  RecordPrinter RP(outs(), "\n");
+  if (!DumpVerify) {
+    PipelineConsumer C({&RP});
+    while (DE.isValidOffsetForDataOfSize(OffsetPtr, 1)) {
+      auto R = P.produce();
+      if (!R)
+        return R.takeError();
+      if (auto E = C.consume(std::move(R.get())))
+        return E;
+    }
+    return Error::success();
+  }
+
+  BlockPrinter BP(outs(), RP);
+  std::vector<std::unique_ptr<Record>> Records;
+  LogBuilderConsumer C(Records);
+  while (DE.isValidOffsetForDataOfSize(OffsetPtr, 1)) {
+    auto R = P.produce();
+    if (!R) {
+      // Print records we've found so far.
+      for (auto &Ptr : Records)
+        if (auto E = Ptr->apply(RP))
+          return joinErrors(std::move(E), R.takeError());
+      return R.takeError();
+    }
+    if (auto E = C.consume(std::move(R.get())))
+      return E;
+  }
+
+  // Once we have a trace, we then index the blocks.
+  BlockIndexer::Index Index;
+  BlockIndexer BI(Index);
+  for (auto &Ptr : Records)
+    if (auto E = Ptr->apply(BI))
+      return E;
+
+  if (auto E = BI.flush())
+    return E;
+
+  // Then we validate while printing each block.
+  BlockVerifier BV;
+  for (auto ProcessThreadBlocks : Index) {
+    auto &Blocks = ProcessThreadBlocks.second;
+    for (auto &B : Blocks) {
+      for (auto *R : B.Records) {
+        if (auto E = R->apply(BV))
+          return E;
+        if (auto E = R->apply(BP))
+          return E;
+      }
+      BV.reset();
+      BP.reset();
+    }
+  }
+  outs().flush();
+  return Error::success();
+});

llvm/unittests/XRay/CMakeLists.txt

 set(LLVM_LINK_COMPONENTS
   Support
   XRay
   TestingSupport
   )
 
 add_llvm_unittest(XRayTests
-  ProfileTest.cpp
   FDRBlockIndexerTest.cpp
   FDRBlockVerifierTest.cpp
   FDRProducerConsumerTest.cpp
   FDRRecordPrinterTest.cpp
+  FDRRecordsTest.cpp
   FDRTraceWriterTest.cpp
   GraphTest.cpp
+  ProfileTest.cpp
   )
 
 add_dependencies(XRayTests intrinsics_gen)

llvm/unittests/XRay/FDRRecordsTest.cpp

+//===- FDRRecords.cpp - Unit Tests for XRay FDR Record Loading ------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+#include "llvm/XRay/BlockIndexer.h"
+#include "llvm/XRay/BlockPrinter.h"
+#include "llvm/XRay/BlockVerifier.h"
+#include "llvm/XRay/FDRLogBuilder.h"
+#include "llvm/XRay/FDRRecords.h"
+#include "llvm/XRay/RecordPrinter.h"
+
+namespace llvm {
+namespace xray {
+namespace {
+
+using ::testing::Eq;
+using ::testing::Not;
+
+TEST(XRayFDRTest, BuilderAndBlockIndexer) {
+  // We recreate a single block of valid records, then ensure that we find all
+  // of them belonging in the same index. We do this for three blocks, and
+  // ensure we find the same records in the blocks we deduce.
+  auto Block0 = LogBuilder()
+                    .add<BufferExtents>(100)
+                    .add<NewBufferRecord>(1)
+                    .add<WallclockRecord>(1, 1)
+                    .add<PIDRecord>(1)
+                    .add<FunctionRecord>(RecordTypes::ENTER, 1, 1)
+                    .add<FunctionRecord>(RecordTypes::EXIT, 1, 100)
+                    .consume();
+  auto Block1 = LogBuilder()
+                    .add<BufferExtents>(100)
+                    .add<NewBufferRecord>(1)
+                    .add<WallclockRecord>(1, 2)
+                    .add<PIDRecord>(1)
+                    .add<FunctionRecord>(RecordTypes::ENTER, 1, 1)
+                    .add<FunctionRecord>(RecordTypes::EXIT, 1, 100)
+                    .consume();
+  auto Block2 = LogBuilder()
+                    .add<BufferExtents>(100)
+                    .add<NewBufferRecord>(2)
+                    .add<WallclockRecord>(1, 3)
+                    .add<PIDRecord>(1)
+                    .add<FunctionRecord>(RecordTypes::ENTER, 1, 1)
+                    .add<FunctionRecord>(RecordTypes::EXIT, 1, 100)
+                    .consume();
+  BlockIndexer::Index Index;
+  BlockIndexer Indexer(Index);
+  for (auto B : {std::ref(Block0), std::ref(Block1), std::ref(Block2)}) {
+    for (auto &R : B.get())
+      ASSERT_FALSE(errorToBool(R->apply(Indexer)));
+    ASSERT_FALSE(errorToBool(Indexer.flush()));
+  }
+
+  // We have two threads worth of blocks.
+  ASSERT_THAT(Index.size(), Eq(2u));
+  auto T1Blocks = Index.find({1, 1});
+  ASSERT_THAT(T1Blocks, Not(Eq(Index.end())));
+  ASSERT_THAT(T1Blocks->second.size(), Eq(2u));
+  auto T2Blocks = Index.find({1, 2});
+  ASSERT_THAT(T2Blocks, Not(Eq(Index.end())));
+  ASSERT_THAT(T2Blocks->second.size(), Eq(1u));
+}
+
+TEST(XRayFDRTest, BuilderAndBlockVerifier) {
+  auto Block = LogBuilder()
+                   .add<BufferExtents>(48)
+                   .add<NewBufferRecord>(1)
+                   .add<WallclockRecord>(1, 1)
+                   .add<PIDRecord>(1)
+                   .add<NewCPUIDRecord>(1)
+                   .consume();
+  BlockVerifier Verifier;
+  for (auto &R : Block)
+    ASSERT_FALSE(errorToBool(R->apply(Verifier)));
+  ASSERT_FALSE(errorToBool(Verifier.verify()));
+}
+
+TEST(XRayFDRTest, IndexAndVerifyBlocks) {
+  auto Block0 = LogBuilder()
+                    .add<BufferExtents>(64)
+                    .add<NewBufferRecord>(1)
+                    .add<WallclockRecord>(1, 1)
+                    .add<PIDRecord>(1)
+                    .add<NewCPUIDRecord>(1)
+                    .add<FunctionRecord>(RecordTypes::ENTER, 1, 1)
+                    .add<FunctionRecord>(RecordTypes::EXIT, 1, 100)
+                    .consume();
+  auto Block1 = LogBuilder()
+                    .add<BufferExtents>(64)
+                    .add<NewBufferRecord>(1)
+                    .add<WallclockRecord>(1, 1)
+                    .add<PIDRecord>(1)
+                    .add<NewCPUIDRecord>(1)
+                    .add<FunctionRecord>(RecordTypes::ENTER, 1, 1)
+                    .add<FunctionRecord>(RecordTypes::EXIT, 1, 100)
+                    .consume();
+  auto Block2 = LogBuilder()
+                    .add<BufferExtents>(64)
+                    .add<NewBufferRecord>(1)
+                    .add<WallclockRecord>(1, 1)
+                    .add<PIDRecord>(1)
+                    .add<NewCPUIDRecord>(1)
+                    .add<FunctionRecord>(RecordTypes::ENTER, 1, 1)
+                    .add<FunctionRecord>(RecordTypes::EXIT, 1, 100)
+                    .consume();
+
+  // First, index the records in different blocks.
+  BlockIndexer::Index Index;
+  BlockIndexer Indexer(Index);
+  for (auto B : {std::ref(Block0), std::ref(Block1), std::ref(Block2)}) {
+    for (auto &R : B.get())
+      ASSERT_FALSE(errorToBool(R->apply(Indexer)));
+    ASSERT_FALSE(errorToBool(Indexer.flush()));
+  }
+
+  // Next, verify that each block is consistently defined.
+  BlockVerifier Verifier;
+  for (auto &ProcessThreadBlocks : Index) {
+    auto &Blocks = ProcessThreadBlocks.second;
+    for (auto &B : Blocks) {
+      for (auto *R : B.Records)
+        ASSERT_FALSE(errorToBool(R->apply(Verifier)));
+      ASSERT_FALSE(errorToBool(Verifier.verify()));
+      Verifier.reset();
+    }
+  }
+
+  // Then set up the printing mechanisms.
+  std::string Output;
+  raw_string_ostream OS(Output);
+  RecordPrinter RP(OS);
+  BlockPrinter BP(OS, RP);
+  for (auto &ProcessThreadBlocks : Index) {
+    auto &Blocks = ProcessThreadBlocks.second;
+    for (auto &B : Blocks) {
+      for (auto *R : B.Records)
+        ASSERT_FALSE(errorToBool(R->apply(BP)));
+      BP.reset();
+    }
+  }
+
+  OS.flush();
+  EXPECT_THAT(Output, Not(Eq("")));
+}
+
+} // namespace
+} // namespace xray
+} // namespace llvm