[llvm-exegesis] Add instructions to BenchmarkResult Key.

[gchatelet]

Summary

We want llvm-exegesis to explore instructions (effect of initial register values, effect of operand selection). To enable this a BenchmarkResult muststore all the relevant data in its key. This patch starts adding such data. Here we simply allow to store the generated instructions, following patches will add operands and initial values for registers.

[courbet]

tools/llvm-exegesis/lib/BenchmarkResult.h

31

nit: move the class definition to the end of the header as this is more an artifact and therefore less important than the schema..

54

Can you mention that we're not directly getting the name from the sched model to avoid creating a dependency to make testing easier ?

tools/llvm-exegesis/llvm-exegesis.cpp

142

writeYamlOrDie(getBenchmarkResultContext(State), BenchmarkFile); ?

173–174

ditto

[gchatelet]

Updating D47764: [llvm-exegesis] Add instructions to BenchmarkResult Key.

[gchatelet]

tools/llvm-exegesis/llvm-exegesis.cpp

142

As discussed offline, I had to make the reference const and const_cast.

[courbet]

This was submitted.

tools/llvm-exegesis/lib/BenchmarkResult.h

 /// Defines classes to represent measurements and serialize/deserialize them to
 //  Yaml.
 ///
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_TOOLS_LLVM_EXEGESIS_BENCHMARKRESULT_H
 #define LLVM_TOOLS_LLVM_EXEGESIS_BENCHMARKRESULT_H
 
+#include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCInstBuilder.h"
 #include "llvm/Support/YAMLTraits.h"
 #include <limits>
 #include <string>
+#include <unordered_map>
 #include <vector>
 
 namespace exegesis {
 
+struct BenchmarkResultContext; // Forward declaration.
+
 struct InstructionBenchmarkKey {
   // The LLVM opcode name.
-  std::string OpcodeName;
+  std::string OpcodeName; // FIXME: Deprecated, use Instructions below.
+  std::vector<llvm::MCInst> Instructions;
   enum ModeE { Unknown, Latency, Uops };
   ModeE Mode;
   // An opaque configuration, that can be used to separate several benchmarks of
   // the same instruction under different configurations.
   std::string Config;
 };
 
 struct BenchmarkMeasure {
   std::string Key;
   double Value;
   std::string DebugString;
 };
 
 // The result of an instruction benchmark.
 struct InstructionBenchmark {
   InstructionBenchmarkKey Key;
   std::string CpuName;
   std::string LLVMTriple;
   int NumRepetitions = 0;
   std::vector<BenchmarkMeasure> Measurements;
   std::string Error;
   std::string Info;
 
-  static InstructionBenchmark readYamlOrDie(llvm::StringRef Filename);
+  // Read functions.
+  static InstructionBenchmark
+  readYamlOrDie(const BenchmarkResultContext &Context,
+                llvm::StringRef Filename);
+
   static std::vector<InstructionBenchmark>
+  readYamlsOrDie(const BenchmarkResultContext &Context,
+                 llvm::StringRef Filename);
 
-  // Read functions.
-  readYamlsOrDie(llvm::StringRef Filename);
-  void readYamlFrom(llvm::StringRef InputContent);
+  void readYamlFrom(const BenchmarkResultContext &Context,
+                    llvm::StringRef InputContent);
 
   // Write functions, non-const because of YAML traits.
-  void writeYamlTo(llvm::raw_ostream &S);
-  void writeYamlOrDie(const llvm::StringRef Filename);
+  void writeYamlTo(const BenchmarkResultContext &Context, llvm::raw_ostream &S);
+
+  void writeYamlOrDie(const BenchmarkResultContext &Context,
+                      const llvm::StringRef Filename);
 };
 
 //------------------------------------------------------------------------------
 // Utilities to work with Benchmark measures.
 
 // A class that measures stats over benchmark measures.
 class BenchmarkMeasureStats {
 public:

 private:
   std::string Key;
   double SumValues = 0.0;
   int NumValues = 0;
   double MaxValue = std::numeric_limits<double>::min();
   double MinValue = std::numeric_limits<double>::max();
 };
 
+// This context is used when de/serializing InstructionBenchmark to guarantee
+// that Registers and Instructions are human readable and preserved accross
+// different versions of LLVM.
+struct BenchmarkResultContext {
+  BenchmarkResultContext() = default;
+  BenchmarkResultContext(BenchmarkResultContext &&) = default;
+  BenchmarkResultContext &operator=(BenchmarkResultContext &&) = default;
+  BenchmarkResultContext(const BenchmarkResultContext &) = delete;
+  BenchmarkResultContext &operator=(const BenchmarkResultContext &) = delete;
+
+  // Populate Registers and Instruction mapping.
+  void addRegEntry(unsigned RegNo, llvm::StringRef Name);
+  void addInstrEntry(unsigned Opcode, llvm::StringRef Name);
+
+  // Register accessors.
+  llvm::StringRef getRegName(unsigned RegNo) const;
+  unsigned getRegNo(llvm::StringRef Name) const; // 0 is not found.
+
+  // Instruction accessors.
+  llvm::StringRef getInstrName(unsigned Opcode) const;
+  unsigned getInstrOpcode(llvm::StringRef Name) const; // 0 is not found.
+
+private:
+  // Ideally we would like to use MCRegisterInfo and MCInstrInfo but doing so
+  // would make testing harder, instead we create a mapping that we can easily
+  // populate.
+  std::unordered_map<unsigned, llvm::StringRef> InstrOpcodeToName;
+  std::unordered_map<unsigned, llvm::StringRef> RegNoToName;
+  llvm::StringMap<unsigned> InstrNameToOpcode;
+  llvm::StringMap<unsigned> RegNameToNo;
+};
+
 } // namespace exegesis
 
 #endif // LLVM_TOOLS_LLVM_EXEGESIS_BENCHMARKRESULT_H

tools/llvm-exegesis/lib/BenchmarkResult.cpp

 //===-- BenchmarkResult.cpp -------------------------------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "BenchmarkResult.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/FileOutputBuffer.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/raw_ostream.h"
 
 // Defining YAML traits for IO.
 namespace llvm {
 namespace yaml {
 
+// std::vector<llvm::MCInst> will be rendered as a list.
+template <> struct SequenceElementTraits<llvm::MCInst> {
+  static const bool flow = false;
+};
+
+template <> struct ScalarTraits<llvm::MCInst> {
+
+  static void output(const llvm::MCInst &Value, void *Ctx,
+                     llvm::raw_ostream &Out) {
+    assert(Ctx);
+    auto *Context = static_cast<const exegesis::BenchmarkResultContext *>(Ctx);
+    const StringRef Name = Context->getInstrName(Value.getOpcode());
+    assert(!Name.empty());
+    Out << Name;
+  }
+
+  static StringRef input(StringRef Scalar, void *Ctx, llvm::MCInst &Value) {
+    assert(Ctx);
+    auto *Context = static_cast<const exegesis::BenchmarkResultContext *>(Ctx);
+    const unsigned Opcode = Context->getInstrOpcode(Scalar);
+    if (Opcode == 0) {
+      return "Unable to parse instruction";
+    }
+    Value.setOpcode(Opcode);
+    return StringRef();
+  }
+
+  static QuotingType mustQuote(StringRef) { return QuotingType::Single; }
+
+  static const bool flow = true;
+};
+
 // std::vector<exegesis::Measure> will be rendered as a list.
 template <> struct SequenceElementTraits<exegesis::BenchmarkMeasure> {
   static const bool flow = false;
 };
 
 // exegesis::Measure is rendererd as a flow instead of a list.
 // e.g. { "key": "the key", "value": 0123 }
 template <> struct MappingTraits<exegesis::BenchmarkMeasure> {
static void enumeration(IO &Io,
     Io.enumCase(Value, "latency", exegesis::InstructionBenchmarkKey::Latency);
     Io.enumCase(Value, "uops", exegesis::InstructionBenchmarkKey::Uops);
   }
 };
 
 template <> struct MappingTraits<exegesis::InstructionBenchmarkKey> {
   static void mapping(IO &Io, exegesis::InstructionBenchmarkKey &Obj) {
     Io.mapRequired("opcode_name", Obj.OpcodeName);
+    Io.mapRequired("instructions", Obj.Instructions);
     Io.mapRequired("mode", Obj.Mode);
     Io.mapOptional("config", Obj.Config);
   }
 };
 
 template <> struct MappingTraits<exegesis::InstructionBenchmark> {
   static void mapping(IO &Io, exegesis::InstructionBenchmark &Obj) {
     Io.mapRequired("key", Obj.Key);
     Io.mapRequired("cpu_name", Obj.CpuName);
     Io.mapRequired("llvm_triple", Obj.LLVMTriple);
     Io.mapRequired("num_repetitions", Obj.NumRepetitions);
     Io.mapRequired("measurements", Obj.Measurements);
     Io.mapRequired("error", Obj.Error);
     Io.mapOptional("info", Obj.Info);
   }
 };
 
 } // namespace yaml
 } // namespace llvm
 
 LLVM_YAML_IS_DOCUMENT_LIST_VECTOR(exegesis::InstructionBenchmark)
 
 namespace exegesis {
 
+void BenchmarkResultContext::addRegEntry(unsigned RegNo, llvm::StringRef Name) {
+  assert(RegNoToName.find(RegNo) == RegNoToName.end());
+  assert(RegNameToNo.find(Name) == RegNameToNo.end());
+  RegNoToName[RegNo] = Name;
+  RegNameToNo[Name] = RegNo;
+}
+
+llvm::StringRef BenchmarkResultContext::getRegName(unsigned RegNo) const {
+  const auto Itr = RegNoToName.find(RegNo);
+  if (Itr != RegNoToName.end())
+    return Itr->second;
+  return {};
+}
+
+unsigned BenchmarkResultContext::getRegNo(llvm::StringRef Name) const {
+  const auto Itr = RegNameToNo.find(Name);
+  if (Itr != RegNameToNo.end())
+    return Itr->second;
+  return 0;
+}
+
+void BenchmarkResultContext::addInstrEntry(unsigned Opcode,
+                                           llvm::StringRef Name) {
+  assert(InstrOpcodeToName.find(Opcode) == InstrOpcodeToName.end());
+  assert(InstrNameToOpcode.find(Name) == InstrNameToOpcode.end());
+  InstrOpcodeToName[Opcode] = Name;
+  InstrNameToOpcode[Name] = Opcode;
+}
+
+llvm::StringRef BenchmarkResultContext::getInstrName(unsigned Opcode) const {
+  const auto Itr = InstrOpcodeToName.find(Opcode);
+  if (Itr != InstrOpcodeToName.end())
+    return Itr->second;
+  return {};
+}
+
+unsigned BenchmarkResultContext::getInstrOpcode(llvm::StringRef Name) const {
+  const auto Itr = InstrNameToOpcode.find(Name);
+  if (Itr != InstrNameToOpcode.end())
+    return Itr->second;
+  return 0;
+}
+
 template <typename ObjectOrList>
-static ObjectOrList readYamlOrDieCommon(llvm::StringRef Filename) {
+static ObjectOrList readYamlOrDieCommon(const BenchmarkResultContext &Context,
+                                        llvm::StringRef Filename) {
   std::unique_ptr<llvm::MemoryBuffer> MemBuffer = llvm::cantFail(
       llvm::errorOrToExpected(llvm::MemoryBuffer::getFile(Filename)));
-  llvm::yaml::Input Yin(*MemBuffer);
+  // YAML IO requires a mutable pointer to Context but we guarantee to not
+  // modify it.
+  llvm::yaml::Input Yin(*MemBuffer,
+                        const_cast<BenchmarkResultContext *>(&Context));
   ObjectOrList Benchmark;
   Yin >> Benchmark;
   return Benchmark;
 }
 
 InstructionBenchmark
-InstructionBenchmark::readYamlOrDie(llvm::StringRef Filename) {
-  return readYamlOrDieCommon<InstructionBenchmark>(Filename);
+InstructionBenchmark::readYamlOrDie(const BenchmarkResultContext &Context,
+                                    llvm::StringRef Filename) {
+  return readYamlOrDieCommon<InstructionBenchmark>(Context, Filename);
 }
 
 std::vector<InstructionBenchmark>
-InstructionBenchmark::readYamlsOrDie(llvm::StringRef Filename) {
-  return readYamlOrDieCommon<std::vector<InstructionBenchmark>>(Filename);
+InstructionBenchmark::readYamlsOrDie(const BenchmarkResultContext &Context,
+                                     llvm::StringRef Filename) {
+  return readYamlOrDieCommon<std::vector<InstructionBenchmark>>(Context,
+                                                                Filename);
 }
 
-void InstructionBenchmark::writeYamlTo(llvm::raw_ostream &S) {
-  llvm::yaml::Output Yout(S);
+void InstructionBenchmark::writeYamlTo(const BenchmarkResultContext &Context,
+                                       llvm::raw_ostream &S) {
+  // YAML IO requires a mutable pointer to Context but we guarantee to not
+  // modify it.
+  llvm::yaml::Output Yout(S, const_cast<BenchmarkResultContext *>(&Context));
   Yout << *this;
 }
 
-void InstructionBenchmark::readYamlFrom(llvm::StringRef InputContent) {
-  llvm::yaml::Input Yin(InputContent);
+void InstructionBenchmark::readYamlFrom(const BenchmarkResultContext &Context,
+                                        llvm::StringRef InputContent) {
+  // YAML IO requires a mutable pointer to Context but we guarantee to not
+  // modify it.
+  llvm::yaml::Input Yin(InputContent,
+                        const_cast<BenchmarkResultContext *>(&Context));
   Yin >> *this;
 }
 
 // FIXME: Change the API to let the caller handle errors.
-void InstructionBenchmark::writeYamlOrDie(const llvm::StringRef Filename) {
+void InstructionBenchmark::writeYamlOrDie(const BenchmarkResultContext &Context,
+                                          const llvm::StringRef Filename) {
   if (Filename == "-") {
-    writeYamlTo(llvm::outs());
+    writeYamlTo(Context, llvm::outs());
   } else {
     int ResultFD = 0;
     llvm::cantFail(llvm::errorCodeToError(
         openFileForWrite(Filename, ResultFD, llvm::sys::fs::F_Text)));
     llvm::raw_fd_ostream Ostr(ResultFD, true /*shouldClose*/);
-    writeYamlTo(Ostr);
+    writeYamlTo(Context, Ostr);
   }
 }
 
 void BenchmarkMeasureStats::push(const BenchmarkMeasure &BM) {
   if (Key.empty())
     Key = BM.Key;
   assert(Key == BM.Key);
   ++NumValues;
   SumValues += BM.Value;
   MaxValue = std::max(MaxValue, BM.Value);
   MinValue = std::min(MinValue, BM.Value);
 }
 
 } // namespace exegesis

tools/llvm-exegesis/llvm-exegesis.cpp

if (OpcodeIndex > 0)
     return OpcodeIndex;
   // Resolve opcode name -> opcode.
   for (unsigned I = 0, E = MCInstrInfo.getNumOpcodes(); I < E; ++I)
     if (MCInstrInfo.getName(I) == OpcodeName)
       return I;
   llvm::report_fatal_error(llvm::Twine("unknown opcode ").concat(OpcodeName));
 }
 
+static BenchmarkResultContext
+getBenchmarkResultContext(const LLVMState &State) {
+  BenchmarkResultContext Ctx;
+
+  const llvm::MCInstrInfo &InstrInfo = State.getInstrInfo();
+  for (unsigned E = InstrInfo.getNumOpcodes(), I = 0; I < E; ++I)
+    Ctx.addInstrEntry(I, InstrInfo.getName(I).data());
+
+  const llvm::MCRegisterInfo &RegInfo = State.getRegInfo();
+  for (unsigned E = RegInfo.getNumRegs(), I = 0; I < E; ++I)
+    Ctx.addRegEntry(I, RegInfo.getName(I));
+
+  return Ctx;
+}
+
 void benchmarkMain() {
   if (exegesis::pfm::pfmInitialize())
     llvm::report_fatal_error("cannot initialize libpfm");
 
   llvm::InitializeNativeTarget();
   llvm::InitializeNativeTargetAsmPrinter();
 
   // FIXME: Target-specific filter.
void benchmarkMain() {
   case BenchmarkModeE::Analysis:
     llvm_unreachable("not a benchmark");
   }
 
   if (NumRepetitions == 0)
     llvm::report_fatal_error("--num-repetitions must be greater than zero");
 
   Runner->run(GetOpcodeOrDie(State.getInstrInfo()), Filter, NumRepetitions)
-      .writeYamlOrDie(BenchmarkFile);
+      .writeYamlOrDie(getBenchmarkResultContext(State), BenchmarkFile);
   exegesis::pfm::pfmTerminate();
 }
 
 // Prints the results of running analysis pass `Pass` to file `OutputFilename`
 // if OutputFilename is non-empty.
 template <typename Pass>
 static void maybeRunAnalysis(const Analysis &Analyzer, const std::string &Name,
                              const std::string &OutputFilename) {
   if (OutputFilename.empty())
     return;
   if (OutputFilename != "-") {
     llvm::errs() << "Printing " << Name << " results to file '"
                  << OutputFilename << "'\n";
   }
   std::error_code ErrorCode;
   llvm::raw_fd_ostream ClustersOS(OutputFilename, ErrorCode,
                                   llvm::sys::fs::F_RW);
   if (ErrorCode)
     llvm::report_fatal_error("cannot open out file: " + OutputFilename);
   if (auto Err = Analyzer.run<Pass>(ClustersOS))
     llvm::report_fatal_error(std::move(Err));
 }
 
 static void analysisMain() {
+  llvm::InitializeNativeTarget();
+  llvm::InitializeNativeTargetAsmPrinter();
+
   // Read benchmarks.
+  const LLVMState State;
   const std::vector<InstructionBenchmark> Points =
-      InstructionBenchmark::readYamlsOrDie(BenchmarkFile);
+      InstructionBenchmark::readYamlsOrDie(getBenchmarkResultContext(State),
+                                           BenchmarkFile);
   llvm::outs() << "Parsed " << Points.size() << " benchmark points\n";
   if (Points.empty()) {
     llvm::errs() << "no benchmarks to analyze\n";
     return;
   }
   // FIXME: Check that all points have the same triple/cpu.
   // FIXME: Merge points from several runs (latency and uops).
 
-  llvm::InitializeNativeTarget();
-  llvm::InitializeNativeTargetAsmPrinter();
-
   std::string Error;
   const auto *TheTarget =
       llvm::TargetRegistry::lookupTarget(Points[0].LLVMTriple, Error);
   if (!TheTarget) {
     llvm::errs() << "unknown target '" << Points[0].LLVMTriple << "'\n";
     return;
   }
   const auto Clustering = llvm::cantFail(InstructionBenchmarkClustering::create(

unittests/tools/llvm-exegesis/BenchmarkResultTest.cpp

 #include "llvm/ADT/SmallString.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/YAMLTraits.h"
 #include "llvm/Support/raw_ostream.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
+using ::testing::AllOf;
+using ::testing::Eq;
+using ::testing::get;
+using ::testing::Pointwise;
+using ::testing::Property;
+
 namespace exegesis {
 
 bool operator==(const BenchmarkMeasure &A, const BenchmarkMeasure &B) {
   return std::tie(A.Key, A.Value) == std::tie(B.Key, B.Value);
 }
 
+MATCHER(EqMCInst, "") {
+  return get<0>(arg).getOpcode() == get<1>(arg).getOpcode();
+}
+
 namespace {
 
+static constexpr const unsigned kInstrId = 5;
+static constexpr const char kInstrName[] = "Instruction5";
+
 TEST(BenchmarkResultTest, WriteToAndReadFromDisk) {
+  BenchmarkResultContext Ctx;
+  Ctx.addInstrEntry(kInstrId, kInstrName);
+
   InstructionBenchmark ToDisk;
 
   ToDisk.Key.OpcodeName = "name";
+  ToDisk.Key.Instructions.push_back(llvm::MCInstBuilder(kInstrId));
   ToDisk.Key.Mode = InstructionBenchmarkKey::Latency;
   ToDisk.Key.Config = "config";
   ToDisk.CpuName = "cpu_name";
   ToDisk.LLVMTriple = "llvm_triple";
   ToDisk.NumRepetitions = 1;
   ToDisk.Measurements.push_back(BenchmarkMeasure{"a", 1, "debug a"});
   ToDisk.Measurements.push_back(BenchmarkMeasure{"b", 2, ""});
   ToDisk.Error = "error";
   ToDisk.Info = "info";
 
   llvm::SmallString<64> Filename;
   std::error_code EC;
   EC = llvm::sys::fs::createUniqueDirectory("BenchmarkResultTestDir", Filename);
   ASSERT_FALSE(EC);
   llvm::sys::path::append(Filename, "data.yaml");
-
-  ToDisk.writeYamlOrDie(Filename);
+  ToDisk.writeYamlOrDie(Ctx, Filename);
 
   {
     // One-element version.
-    const auto FromDisk = InstructionBenchmark::readYamlOrDie(Filename);
+    const auto FromDisk = InstructionBenchmark::readYamlOrDie(Ctx, Filename);
 
     EXPECT_EQ(FromDisk.Key.OpcodeName, ToDisk.Key.OpcodeName);
+    EXPECT_THAT(FromDisk.Key.Instructions,
+                Pointwise(EqMCInst(), ToDisk.Key.Instructions));
     EXPECT_EQ(FromDisk.Key.Mode, ToDisk.Key.Mode);
     EXPECT_EQ(FromDisk.Key.Config, ToDisk.Key.Config);
     EXPECT_EQ(FromDisk.CpuName, ToDisk.CpuName);
     EXPECT_EQ(FromDisk.LLVMTriple, ToDisk.LLVMTriple);
     EXPECT_EQ(FromDisk.NumRepetitions, ToDisk.NumRepetitions);
     EXPECT_THAT(FromDisk.Measurements, ToDisk.Measurements);
     EXPECT_THAT(FromDisk.Error, ToDisk.Error);
     EXPECT_EQ(FromDisk.Info, ToDisk.Info);
   }
   {
     // Vector version.
-    const auto FromDiskVector = InstructionBenchmark::readYamlsOrDie(Filename);
+    const auto FromDiskVector =
+        InstructionBenchmark::readYamlsOrDie(Ctx, Filename);
     ASSERT_EQ(FromDiskVector.size(), size_t{1});
     const auto FromDisk = FromDiskVector[0];
     EXPECT_EQ(FromDisk.Key.OpcodeName, ToDisk.Key.OpcodeName);
+    EXPECT_THAT(FromDisk.Key.Instructions,
+                Pointwise(EqMCInst(), ToDisk.Key.Instructions));
     EXPECT_EQ(FromDisk.Key.Mode, ToDisk.Key.Mode);
     EXPECT_EQ(FromDisk.Key.Config, ToDisk.Key.Config);
     EXPECT_EQ(FromDisk.CpuName, ToDisk.CpuName);
     EXPECT_EQ(FromDisk.LLVMTriple, ToDisk.LLVMTriple);
     EXPECT_EQ(FromDisk.NumRepetitions, ToDisk.NumRepetitions);
     EXPECT_THAT(FromDisk.Measurements, ToDisk.Measurements);
     EXPECT_THAT(FromDisk.Error, ToDisk.Error);
     EXPECT_EQ(FromDisk.Info, ToDisk.Info);